From 620e29f69718b32e7d988b0abb85cb26aa95475b Mon Sep 17 00:00:00 2001 From: dranion Date: Sun, 23 Aug 2020 14:23:04 -0700 Subject: [PATCH] updated to python3 and debugged --- .gitignore | 1 + .idea/.gitignore | 8 + .../inspectionProfiles/profiles_settings.xml | 6 + .idea/misc.xml | 7 + .idea/modules.xml | 8 + .idea/protect3.iml | 18 + .idea/runConfigurations/Basic_Run_.xml | 13 + .idea/vcs.xml | 6 + MANUAL.md | 6 +- Makefile | 15 +- ProTECT_config.yaml | 169 + README.md | 5 +- attic/ProTECT.py | 52 +- attic/ProTECT_large.py | 50 +- attic/encrypt_files_in_dir_to_s3.py | 4 +- attic/precision_immuno.py | 50 +- changes.md | 22 + docker/pipelineWrapper.py | 6 +- docker/wrapper.py | 12 +- errors/2020-10-19-14-57-49.txt | 438 ++ errors/2020-10-19-14.txt | 10 + errors/2020-10-19-16-54-58.txt | 12 + errors/2020-10-19-16-55-14.txt | 613 +++ errors/2020-10-20-19-53-59.txt | 65 + errors/2020-10-20-19-55-00.txt | 1387 +++++++ errors/2020-10-20-20-18-18.txt | 0 errors/2020-10-20-20-40-18.txt | 115 + errors/2020-10-22-17-18-33.txt | 33 + errors/2020-10-22-17-19-05.txt | 12 + errors/2020-10-22-17-19-15.txt | 12 + errors/2020-10-22-17-19-23.txt | 121 + errors/2020-10-31-03-14-30.txt | 197 + errors/2020-10-31-03-14-53.txt | 263 ++ errors/2020-11-08-15-44-22.txt | 76 + errors/2020-11-08-15-45-24.txt | 78 + errors/2020-11-10-17-39-03.txt | 43 + errors/2020-11-10-17-47-22.txt | 25 + errors/2020-11-10-17-47-50.txt | 46 + errors/2020-11-10-17-48-39.txt | 19 + errors/2020-11-10-17-48-52.txt | 263 ++ errors/2020-11-11-13-41-03.txt | 114 + errors/2020-11-11-16-09-15.txt | 113 + errors/2020-11-13-15-32-47.txt | 12 + errors/2020-11-13-15-33-17.txt | 222 ++ errors/2020-11-13-16-34-54.txt | 19 + errors/2020-11-13-16-35-47.txt | 252 ++ errors/2020-11-15-00-28-09.txt | 720 ++++ errors/2020-11-17-10-01-21.txt | 19 + errors/2020-11-17-10-01-52.txt | 810 ++++ errors/2020-11-17-11-07-31.txt | 12 + errors/2020-11-17-11-08-36.txt | 3537 +++++++++++++++++ errors/2020-11-17-14-13-19.txt | 1749 ++++++++ errors/2020-11-24-00-22-56.txt | 168 + errors/2020-11-24-00-26-36.txt | 17 + errors/2020-11-24-00-27-18.txt | 17 + errors/2020-11-24-00-28-09.txt | 166 + errors/2020-11-24-00-47-40.txt | 62 + errors/2020-11-24-00-52-41.txt | 130 + errors/2020-11-24-00-54-50.txt | 130 + errors/2020-11-24-00-56-41.txt | 266 ++ errors/2020-11-24-17-00-21.txt | 408 ++ errors/2020-11-25-14-55-40.txt | 307 ++ errors/2020-11-25-16-10-09.txt | 720 ++++ errors/2020-11-25-20-22-32.txt | 17 + errors/2020-11-25-20-23-15.txt | 267 ++ errors/2020-11-25-20-26-48.txt | 105 + errors/2020-11-26-14-11-04.txt | 1989 +++++++++ errors/2020-11-27-14-10-34.txt | 718 ++++ errors/2020-11-27-15-11-20.txt | 19 + errors/2020-11-27-15-11-35.txt | 1813 +++++++++ errors/2020-11-27-17-28-31.txt | 103 + errors/2020-11-27-17-32-35.txt | 52 + errors/2020-11-27-17-47-44.txt | 52 + errors/2020-11-27-18-37-04.txt | 52 + errors/2020-11-27-18-42-33.txt | 1577 ++++++++ errors/2020-11-27-18-51-38.txt | 808 ++++ errors/2020-11-27-20-37-31.txt | 246 ++ errors/202010101748 Insufficient Resources | 379 ++ errors/202010102013 | 260 ++ errors/202010110258 phlat | 355 ++ errors/20201019 actualerror | 18 + mustard_config.yaml | 179 + run.sh | 3 + setup.py | 41 +- src/__init__.py | 4 +- src/protect/ProTECT_config.yaml | 179 + src/protect/__init__.py | 4 +- src/protect/addons/__init__.py | 4 +- src/protect/addons/assess_car_t_validity.py | 4 +- .../addons/assess_immunotherapy_resistance.py | 4 +- src/protect/addons/assess_mhc_pathway.py | 12 +- src/protect/addons/common.py | 4 +- src/protect/alignment/__init__.py | 4 +- src/protect/alignment/common.py | 4 +- src/protect/alignment/dna.py | 6 +- src/protect/alignment/rna.py | 6 +- src/protect/binding_prediction/__init__.py | 4 +- src/protect/binding_prediction/common.py | 18 +- src/protect/binding_prediction/mhci.py | 4 +- src/protect/binding_prediction/mhcii.py | 4 +- src/protect/common.py | 61 +- src/protect/expression_profiling/__init__.py | 4 +- src/protect/expression_profiling/rsem.py | 8 +- src/protect/haplotyping/__init__.py | 4 +- src/protect/haplotyping/phlat.py | 15 +- src/protect/mutation_annotation/__init__.py | 4 +- src/protect/mutation_annotation/snpeff.py | 6 +- src/protect/mutation_calling/__init__.py | 4 +- src/protect/mutation_calling/common.py | 24 +- src/protect/mutation_calling/fusion.py | 27 +- src/protect/mutation_calling/indel.py | 4 +- src/protect/mutation_calling/muse.py | 10 +- src/protect/mutation_calling/mutect.py | 19 +- src/protect/mutation_calling/radia.py | 10 +- src/protect/mutation_calling/somaticsniper.py | 10 +- src/protect/mutation_calling/strelka.py | 6 +- src/protect/mutation_translation.py | 10 +- src/protect/pipeline/ProTECT.py | 33 +- src/protect/pipeline/__init__.py | 4 +- src/protect/qc/__init__.py | 4 +- src/protect/qc/rna.py | 6 +- src/protect/rankboost.py | 2 +- src/protect/test/__init__.py | 4 +- src/protect/test/ci/__init__.py | 4 +- src/protect/test/ci/test_protect.py | 12 +- src/protect/test/unit/__init__.py | 4 +- src/protect/test/unit/test_alignments.py | 8 +- src/protect/test/unit/test_file_downloads.py | 4 +- .../test/unit/test_mutation_callers.py | 10 +- src/protect/test/unit/test_rankboost.py | 4 +- src/protect/test/unit/test_reporting.py | 4 +- src/protect/test/unit/test_snpeff.py | 4 +- .../unit/test_spawn_antigen_predictors.py | 4 +- src/protect/test/unit/test_transgene | 2 +- src/protect/version.py | 2 +- 135 files changed, 23597 insertions(+), 318 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/protect3.iml create mode 100644 .idea/runConfigurations/Basic_Run_.xml create mode 100644 .idea/vcs.xml mode change 100644 => 100755 Makefile create mode 100755 ProTECT_config.yaml create mode 100644 changes.md create mode 100644 errors/2020-10-19-14-57-49.txt create mode 100644 errors/2020-10-19-14.txt create mode 100644 errors/2020-10-19-16-54-58.txt create mode 100644 errors/2020-10-19-16-55-14.txt create mode 100644 errors/2020-10-20-19-53-59.txt create mode 100644 errors/2020-10-20-19-55-00.txt create mode 100644 errors/2020-10-20-20-18-18.txt create mode 100644 errors/2020-10-20-20-40-18.txt create mode 100755 errors/2020-10-22-17-18-33.txt create mode 100755 errors/2020-10-22-17-19-05.txt create mode 100755 errors/2020-10-22-17-19-15.txt create mode 100755 errors/2020-10-22-17-19-23.txt create mode 100755 errors/2020-10-31-03-14-30.txt create mode 100755 errors/2020-10-31-03-14-53.txt create mode 100644 errors/2020-11-08-15-44-22.txt create mode 100644 errors/2020-11-08-15-45-24.txt create mode 100644 errors/2020-11-10-17-39-03.txt create mode 100644 errors/2020-11-10-17-47-22.txt create mode 100644 errors/2020-11-10-17-47-50.txt create mode 100644 errors/2020-11-10-17-48-39.txt create mode 100644 errors/2020-11-10-17-48-52.txt create mode 100644 errors/2020-11-11-13-41-03.txt create mode 100644 errors/2020-11-11-16-09-15.txt create mode 100644 errors/2020-11-13-15-32-47.txt create mode 100644 errors/2020-11-13-15-33-17.txt create mode 100644 errors/2020-11-13-16-34-54.txt create mode 100644 errors/2020-11-13-16-35-47.txt create mode 100644 errors/2020-11-15-00-28-09.txt create mode 100644 errors/2020-11-17-10-01-21.txt create mode 100644 errors/2020-11-17-10-01-52.txt create mode 100644 errors/2020-11-17-11-07-31.txt create mode 100644 errors/2020-11-17-11-08-36.txt create mode 100644 errors/2020-11-17-14-13-19.txt create mode 100644 errors/2020-11-24-00-22-56.txt create mode 100644 errors/2020-11-24-00-26-36.txt create mode 100644 errors/2020-11-24-00-27-18.txt create mode 100644 errors/2020-11-24-00-28-09.txt create mode 100644 errors/2020-11-24-00-47-40.txt create mode 100644 errors/2020-11-24-00-52-41.txt create mode 100644 errors/2020-11-24-00-54-50.txt create mode 100644 errors/2020-11-24-00-56-41.txt create mode 100644 errors/2020-11-24-17-00-21.txt create mode 100644 errors/2020-11-25-14-55-40.txt create mode 100644 errors/2020-11-25-16-10-09.txt create mode 100644 errors/2020-11-25-20-22-32.txt create mode 100644 errors/2020-11-25-20-23-15.txt create mode 100644 errors/2020-11-25-20-26-48.txt create mode 100644 errors/2020-11-26-14-11-04.txt create mode 100644 errors/2020-11-27-14-10-34.txt create mode 100644 errors/2020-11-27-15-11-20.txt create mode 100644 errors/2020-11-27-15-11-35.txt create mode 100644 errors/2020-11-27-17-28-31.txt create mode 100644 errors/2020-11-27-17-32-35.txt create mode 100644 errors/2020-11-27-17-47-44.txt create mode 100644 errors/2020-11-27-18-37-04.txt create mode 100644 errors/2020-11-27-18-42-33.txt create mode 100644 errors/2020-11-27-18-51-38.txt create mode 100644 errors/2020-11-27-20-37-31.txt create mode 100644 errors/202010101748 Insufficient Resources create mode 100644 errors/202010102013 create mode 100644 errors/202010110258 phlat create mode 100644 errors/20201019 actualerror create mode 100755 mustard_config.yaml create mode 100755 run.sh mode change 100644 => 100755 setup.py create mode 100644 src/protect/ProTECT_config.yaml mode change 100644 => 100755 src/protect/common.py diff --git a/.gitignore b/.gitignore index 3288e17f..fd021fe6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ pimmuno_2.py develop_data/ venv/ .cache/ +jobStore/ test-report.xml __pycache__ *.DONE diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 00000000..73f69e09 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 00000000..105ce2da --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 00000000..0d03c0f7 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 00000000..88979df1 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/protect3.iml b/.idea/protect3.iml new file mode 100644 index 00000000..7d54b365 --- /dev/null +++ b/.idea/protect3.iml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations/Basic_Run_.xml b/.idea/runConfigurations/Basic_Run_.xml new file mode 100644 index 00000000..2dfd7bf3 --- /dev/null +++ b/.idea/runConfigurations/Basic_Run_.xml @@ -0,0 +1,13 @@ + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..94a25f7f --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/MANUAL.md b/MANUAL.md index b023baad..b7b04814 100644 --- a/MANUAL.md +++ b/MANUAL.md @@ -547,7 +547,7 @@ purposes: 12: g/f/jobO4yiE4 return self.run(fileStore) 13: g/f/jobO4yiE4 File "/home/ucsc/arjun/tools/dev/toil_clean/src/toil/job.py", line 1406, in run 14: g/f/jobO4yiE4 rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) - 15: g/f/jobO4yiE4 File "/home/ucsc/arjun/tools/protect_toil_clean/local/lib/python2.7/site-packages/protect/binding_prediction/common.py", line 566, in merge_mhc_peptide_calls + 15: g/f/jobO4yiE4 File "/home/ucsc/arjun/tools/protect_toil_clean/local/lib/python3/site-packages/protect/binding_prediction/common.py", line 566, in merge_mhc_peptide_calls 16: g/f/jobO4yiE4 raise RuntimeError('No peptides available for ranking') 17: g/f/jobO4yiE4 RuntimeError: No peptides available for ranking 18: g/f/jobO4yiE4 ERROR:toil.worker:Exiting the worker because of a failed job on host sjcb10st7 @@ -581,9 +581,9 @@ do not store logs from tools (see BD2KGenomics/protect#275). The error looks sim Z/O/job1uH92D return self.run(fileStore) Z/O/job1uH92D File "/home/ucsc/arjun/tools/dev/toil_clean/src/toil/job.py", line 1406, in run Z/O/job1uH92D rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) - Z/O/job1uH92D File "/home/ucsc/arjun/tools/protect_toil_clean/local/lib/python2.7/site-packages/protect/mutation_calling/radia.py", line 238, in run_filter_radia + Z/O/job1uH92D File "/home/ucsc/arjun/tools/protect_toil_clean/local/lib/python3/site-packages/protect/mutation_calling/radia.py", line 238, in run_filter_radia Z/O/job1uH92D tool_version=radia_options['version']) - Z/O/job1uH92D File "/home/ucsc/arjun/tools/protect_toil_clean/local/lib/python2.7/site-packages/protect/common.py", line 138, in docker_call + Z/O/job1uH92D File "/home/ucsc/arjun/tools/protect_toil_clean/local/lib/python3/site-packages/protect/common.py", line 138, in docker_call Z/O/job1uH92D 'for command \"%s\"' % ' '.join(call),) Z/O/job1uH92D RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /scratch/bio/ucsc/toil-681c097c-61da-4687-b734-c5051f0aa19f/tmped2fnu/f041f939-5c0d-40be-a884-68635e929d09:/data --log-driver=none aarjunrao/filterradia:bcda721fc1f9c28d8b9224c2f95c440759cd3a03 TCGA-CH-5788 17 /data/radia.vcf /data /home/radia/scripts -d /data/radia_dbsnp -r /data/radia_retrogenes -p /data/radia_pseudogenes -c /data/radia_cosmic -t /data/radia_gencode --noSnpEff --noBlacklist --noTargets --noRnaBlacklist -f /data/hg38.fa --log=INFO -g /data/radia_filtered_chr17_radia.log" Z/O/job1uH92D ERROR:toil.worker:Exiting the worker because of a failed job on host sjcb10st1 diff --git a/Makefile b/Makefile old mode 100644 new mode 100755 index a5a43cb2..d006a5bd --- a/Makefile +++ b/Makefile @@ -45,17 +45,16 @@ help: @echo "$$help" -python=python2.7 -pip=pip2.7 +python=python +pip=pip tests=src/protect/test/unit extras= - green=\033[0;32m normal=\033[0m red=\033[0;31m prepare: check_venv - @$(pip) install toil==3.8.0 pytest==2.8.3 + @$(pip) install toil pytest develop: check_venv $(pip) install -e .$(extras) @@ -107,10 +106,10 @@ clean_pypi: clean: clean_develop clean_sdist clean_pypi - -check_venv: - @$(python) -c 'import sys; sys.exit( int( not hasattr(sys, "real_prefix") ) )' \ - || ( echo "$(red)A virtualenv must be active.$(normal)" ; false ) +#always fails, even though in a venv +#check_venv: +# @$(python) -c 'import sys; sys.exit( int( not hasattr(sys, "real_prefix") ) )' \ +# || ( echo "$(red)A virtualenv must be active.$(normal)" ; false ) check_clean_working_copy: diff --git a/ProTECT_config.yaml b/ProTECT_config.yaml new file mode 100755 index 00000000..2f626712 --- /dev/null +++ b/ProTECT_config.yaml @@ -0,0 +1,169 @@ +## Copyright 2016 UCSC Computational Genomics Lab +## Original contributor: Arjun Arkal Rao +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +#################################################################################################### +#################################################################################################### +## This is the input parameter file for the precision immuno pipeline. The parameters for each of +## the tools is provided here. The file is written in the YAML format. A nice description of the +## format can be found at http://docs.ansible.com/ansible/YAMLSyntax.html +## +## You can add comments anywhere in this file by prefixing it with a '#' +## +## Unless otherwise mentioned, all fields must be filled. +## +#################################################################################################### +#################################################################################################### + +# Any number of patients/samples can be listed here +patients: + TEST: + tumor_dna_fastq_1: /home/drkthomp/gerald_D1VCPACXX_1.bam + normal_dna_fastq_2: /home/drkthomp/gerald_D1VCPACXX_6.bam + tumor_rna_fastq_2: /home/drkthomp//gerald_C1TD1ACXX_8_ACAGTG.bam + tumor_type: 'SKCM' + +# These are options that are used by most tools +Universal_Options: + dockerhub: aarjunrao + java_Xmx: 12G + reference_build: hg38 # Acceptable options are hg38, hg38, GRCh37, GRCh38 + # sse_key: /path/to/master.key # Path to the AWS master key. Required if using AWS else optional + # sse_key_is_master: True # True or False. Required if using AWS else optional + # gdc_download_token: /path/to/token.txt # Path to the user's GDC download token. + storage_location: Local # Local or aws: for where the output must go + #storage_location: aws:protect-run-xyz + output_folder: /home/drkthomp/results # Path to where the output must go. + #mail_to: test.email@host.com # Email patients: + + +# for sending success report. + + +# These options are for each module. You probably don't need to change any of this! +alignment: + cutadapt: + a: AGATCGGAAGAG + A: AGATCGGAAGAG + # version: 1.9.1 + star: + type: star # use starlong if your reads are > 150bp + index: /home/drkthomp/e/protect-data/star_with_fusion_100bp_readlen_indexes.tar.gz # Use star_without if you set star_fusion = False + # version: 2.5.2b + bwa: + index: /home/drkthomp/e/protect-data/bwa_index.tar.gz + # version: 0.7.9a + post: + samtools: + # version: 1.2 + picard: + # version: 1.135 + +expression_estimation: + rsem: + index: /home/drkthomp/e/protect-data/rsem_index.tar.gz + # version: 1.2.0 + +mutation_calling: + indexes: + chromosomes: canonical_chr, chrM + genome_fasta: /home/drkthomp/e/protect-data/hg38.fa.tar.gz + genome_fai: /home/drkthomp/e/protect-data/hg38.fa.fai.tar.gz + genome_dict: /home/drkthomp/e/protect-data/hg38.dict.tar.gz + cosmic_vcf: /home/drkthomp/e/protect-data/CosmicCodingMuts.vcf.tar.gz + cosmic_idx: /home/drkthomp/e/protect-data/CosmicCodingMuts.vcf.idx.tar.gz + dbsnp_vcf: /home/drkthomp/e/protect-data/dbsnp_coding.vcf.gz + dbsnp_idx: /home/drkthomp/e/protect-data/dbsnp_coding.vcf.idx.tar.gz + dbsnp_tbi: /home/drkthomp/e/protect-data/dbsnp_coding.vcf.gz.tbi + mutect: + java_Xmx: 2G + # version: 1.1.7 + muse: + # version: 1.0rc_submission_b391201 + radia: + cosmic_beds: /home/drkthomp/e/protect-data/radia_cosmic.tar.gz + dbsnp_beds: /home/drkthomp/e/protect-data/radia_dbsnp.tar.gz + retrogene_beds: /home/drkthomp/e/protect-data/radia_retrogenes.tar.gz + pseudogene_beds: /home/drkthomp/e/protect-data/radia_pseudogenes.tar.gz + gencode_beds: /home/drkthomp/e/protect-data/radia_gencode.tar.gz + # version: 398366ef07b5911d8082ed61cbf03d487a41f286 + somaticsniper: + # version: 1.0.4 + samtools: + # version: 0.1.8 + bam_readcount: + # version: 0.7.4 + star_fusion: + #run: True + #version: 1.0.0 + fusion_inspector: + #run_trinity: True + #version: 1.0.1 + strelka: + # version: 1.0.15 + config_file: /home/drkthomp/e/protect-data/strelka_bwa_WXS_config.ini.tar.gz + + +mutation_annotation: + snpeff: + index: /home/drkthomp/e/protect-data/snpeff_index.tar.gz + # version: 3.6 + java_Xmx: 20G + +mutation_translation: + transgene: + gencode_peptide_fasta : /home/drkthomp/e/protect-data/gencode.v25.pc_translations_NOPARY.fa.tar.gz + gencode_transcript_fasta : /home/drkthomp/e/protect-data/gencode.v25.pc_transcripts_NOPARY.fa.tar.gz + gencode_annotation_gtf : /home/drkthomp/e/protect-data/gencode.v25.annotation_NOPARY.gtf.tar.gz + genome_fasta : /home/drkthomp/e/protect-data/hg38.fa.tar.gz + # version: 2.2.2 + +haplotyping: + phlat: + index: /home/drkthomp/e/protect-data/phlat_index.tar.gz + # version: 1.0 + +mhc_peptide_binding: + mhci: + method_file: /home/drkthomp/e/protect-data/mhci_restrictions.json.tar.gz + pred: IEDB_recommended + # version: 2.13 + mhcii: + method_file: /home/drkthomp/e/protect-data/mhcii_restrictions.json.tar.gz + pred: IEDB_recommended + # version: 2.13 + netmhciipan: + # version: 3.1 + +prediction_ranking: + rankboost: + mhci_args: + npa: 0.0 + nph: 0.0 + nMHC: 0.32 + TPM: 0.0 + overlap: 0.68 + tndelta: 0.0 + mhcii_args: + npa: 0.2 + nph: 0.2 + nMHC: 0.2 + TPM: 0.2 + tndelta: 0.2 + # version: 2.0.3 + +reports: + mhc_pathways_file: /home/drkthomp/e/protect-data/mhc_pathways.tsv.tar.gz + itx_resistance_file: /home/drkthomp/e/protect-data/itx_resistance.tsv.tar.gz + immune_resistance_pathways_file: /home/drkthomp/e/protect-data/immune_resistance_pathways.json.tar.gz + car_t_targets_file: /home/drkthomp/e/protect-data/car_t_targets.tsv.tar.gz diff --git a/README.md b/README.md index c7306f5a..48f1bac2 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,10 @@ -[![Stories in Ready](https://badge.waffle.io/BD2KGenomics/protect.png?label=ready&title=Ready)](https://waffle.io/BD2KGenomics/protect) # ProTECT ### **Pr**ediction **o**f **T**-Cell **E**pitopes for **C**ancer **T**herapy +Adapation of ProTECT to use python 3.8 instead of 2.7. Currently have tested a complete run using fastq files from [HCC1395 WGS Exome RNA Seq Data](https://github.com/genome/gms/wiki/HCC1395-WGS-Exome-RNA-Seq-Data), but have not checked results against the [original ProTECT](https://github.com/BD2KGenomics/protect) with TCGA PRAD yet. + +Adaptation done using 2to3 and manual bug testing. Manual changes recorded [at changes.md](https://github.com/Dranion/protect/blob/master/changes.md). Since s3am is python2, **currently is local only**, however an untested python3 version of s3am exists [here](https://github.com/Dranion/bd2k-extras/tree/main). Continuing to the original README: + This repo contains the Python libraries for the Precision Immunology Pipeline developed at UCSC. src/protect/pipeline/ProTECT.py - The python script for running the pipeline. diff --git a/attic/ProTECT.py b/attic/ProTECT.py index d8ae8690..a3953145 100644 --- a/attic/ProTECT.py +++ b/attic/ProTECT.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 Arjun Arkal Rao # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,7 @@ Program info can be found in the docstring of the main function. Details can also be obtained by running the script with -h . """ -from __future__ import print_function + import argparse import errno @@ -33,7 +33,7 @@ import time from collections import defaultdict, Counter from multiprocessing import cpu_count -from urlparse import urlparse +from urllib.parse import urlparse from pysam import Samfile @@ -78,7 +78,7 @@ def parse_config_file(job, config_file): # along with it's parameters. for groupname, group_params in tool_specific_param_generator(job, conf): if groupname == 'patient': - if 'patient_id' not in group_params.keys(): + if 'patient_id' not in list(group_params.keys()): raise ParameterError('A patient group is missing the patient_id flag.') sample_set[group_params['patient_id']] = group_params elif groupname == 'Universal_Options': @@ -104,7 +104,7 @@ def parse_config_file(job, config_file): raise ParameterError(' The following tools have no arguments in the config file : \n' + '\n'.join(missing_tools)) # Start a job for each sample in the sample set - for patient_id in sample_set.keys(): + for patient_id in list(sample_set.keys()): job.addFollowOnJobFn(pipeline_launchpad, sample_set[patient_id], univ_options, tool_options) return None @@ -248,7 +248,7 @@ def delete_fastqs(job, fastqs): +- 'normal_dna': [ , ] """ for fq_type in ['tumor_rna', 'tumor_dna', 'normal_dna']: - for i in xrange(0,2): + for i in range(0,2): job.fileStore.deleteGlobalFile(fastqs[fq_type][i]) return None @@ -727,7 +727,7 @@ def spawn_radia(job, rna_bam, tumor_bam, normal_bam, univ_options, radia_options 'normal_dnai': normal_bam['normal_dna_fix_pg_sorted.bam.bai']} # Make a dict object to hold the return values for each of the chromosome jobs. Then run radia # on each chromosome. - chromosomes = [''.join(['chr', str(x)]) for x in range(1, 23) + ['X', 'Y']] + chromosomes = [''.join(['chr', str(x)]) for x in list(range(1, 23)) + ['X', 'Y']] perchrom_radia = defaultdict() for chrom in chromosomes: perchrom_radia[chrom] = job.addChildJobFn(run_radia, bams, univ_options, radia_options, @@ -755,11 +755,11 @@ def merge_radia(job, perchrom_rvs): work_dir = job.fileStore.getLocalTempDir() # We need to squash the input dict of dicts to a single dict such that it can be passed to # get_files_from_filestore - input_files = {filename: jsid for perchrom_files in perchrom_rvs.values() - for filename, jsid in perchrom_files.items()} + input_files = {filename: jsid for perchrom_files in list(perchrom_rvs.values()) + for filename, jsid in list(perchrom_files.items())} input_files = get_files_from_filestore(job, input_files, work_dir, docker=False) - chromosomes = [''.join(['chr', str(x)]) for x in range(1, 23) + ['X', 'Y']] + chromosomes = [''.join(['chr', str(x)]) for x in list(range(1, 23)) + ['X', 'Y']] with open('/'.join([work_dir, 'radia_calls.vcf']), 'w') as radfile, \ open('/'.join([work_dir, 'radia_filter_passing_calls.vcf']), 'w') as radpassfile: for chrom in chromosomes: @@ -961,7 +961,7 @@ def spawn_mutect(job, tumor_bam, normal_bam, univ_options, mutect_options): job.fileStore.logToMaster('Running spawn_mutect on %s' % univ_options['patient']) # Make a dict object to hold the return values for each of the chromosome # jobs. Then run mutect on each chromosome. - chromosomes = [''.join(['chr', str(x)]) for x in range(1, 23) + ['X', 'Y']] + chromosomes = [''.join(['chr', str(x)]) for x in list(range(1, 23)) + ['X', 'Y']] perchrom_mutect = defaultdict() for chrom in chromosomes: perchrom_mutect[chrom] = job.addChildJobFn(run_mutect, tumor_bam, normal_bam, univ_options, @@ -987,10 +987,10 @@ def merge_mutect(job, perchrom_rvs): work_dir = job.fileStore.getLocalTempDir() # We need to squash the input dict of dicts to a single dict such that it can be passed to # get_files_from_filestore - input_files = {filename: jsid for perchrom_files in perchrom_rvs.values() - for filename, jsid in perchrom_files.items()} + input_files = {filename: jsid for perchrom_files in list(perchrom_rvs.values()) + for filename, jsid in list(perchrom_files.items())} input_files = get_files_from_filestore(job, input_files, work_dir, docker=False) - chromosomes = [''.join(['chr', str(x)]) for x in range(1, 23) + ['X', 'Y']] + chromosomes = [''.join(['chr', str(x)]) for x in list(range(1, 23)) + ['X', 'Y']] with open('/'.join([work_dir, 'mutect_calls.vcf']), 'w') as mutvcf, \ open('/'.join([work_dir, 'mutect_calls.out']), 'w') as mutout, \ open('/'.join([work_dir, 'mutect_passing_calls.vcf']), 'w') as mutpassvcf: @@ -1139,7 +1139,7 @@ def run_mutation_aggregator(job, fusion_output, radia_output, mutect_output, ind input_files.pop('fusion.vcf') # read files into memory vcf_file = defaultdict() - mutcallers = input_files.keys() + mutcallers = list(input_files.keys()) with open(''.join([work_dir, '/', univ_options['patient'], '_merged_mutations.vcf']), 'w') as merged_mut_file: for mut_caller in mutcallers: @@ -1571,8 +1571,8 @@ def merge_mhc_peptide_calls(job, antigen_predictions, transgened_files): mhci_files = get_files_from_filestore(job, mhci_preds, work_dir) # First split mhcii_preds into prediction files and predictors and maintain keys so we can later # reference them in pairs - mhcii_predictors = {x: y[1] for x, y in mhcii_preds.items()} - mhcii_files = {x: y[0] for x, y in mhcii_preds.items()} + mhcii_predictors = {x: y[1] for x, y in list(mhcii_preds.items())} + mhcii_files = {x: y[0] for x, y in list(mhcii_preds.items())} mhcii_files = get_files_from_filestore(job, mhcii_files, work_dir) # Get peptide files pept_files = get_files_from_filestore(job, pept_files, work_dir) @@ -1584,7 +1584,7 @@ def merge_mhc_peptide_calls(job, antigen_predictions, transgened_files): pepmap = json.load(mapfile) # Incorporate peptide names into the merged calls with open('/'.join([work_dir, 'mhci_merged_files.list']), 'w') as mhci_resfile: - for mhcifile in mhci_files.values(): + for mhcifile in list(mhci_files.values()): with open(mhcifile, 'r') as mf: for line in mf: # Skip header lines @@ -1605,7 +1605,7 @@ def merge_mhc_peptide_calls(job, antigen_predictions, transgened_files): # Incorporate peptide names into the merged calls with open('/'.join([work_dir, 'mhcii_merged_files.list']), 'w') as \ mhcii_resfile: - for mhciifile in mhcii_files.keys(): + for mhciifile in list(mhcii_files.keys()): core_col = None # Variable to hold the column number with the core if mhcii_predictors[mhciifile] == 'Consensus': with open(mhcii_files[mhciifile], 'r') as mf: @@ -1814,7 +1814,7 @@ def prepare_samples(job, fastqs, univ_options): 'normal_dna_fastq_prefix'} if set(fastqs.keys()).difference(allowed_samples) != {'patient_id'}: raise ParameterError('Sample with the following parameters has an error:\n' + - '\n'.join(fastqs.values())) + '\n'.join(list(fastqs.values()))) # For each sample type, check if the prefix is an S3 link or a regular file # Download S3 files. for sample_type in ['tumor_dna', 'tumor_rna', 'normal_dna']: @@ -1877,7 +1877,7 @@ def get_files_from_filestore(job, files, work_dir, cache=True, docker=False): work_dir is the location where the file should be stored cache indiciates whether caching should be used """ - for name in files.keys(): + for name in list(files.keys()): outfile = job.fileStore.readGlobalFile(files[name], '/'.join([work_dir, name]), cache=cache) # If the file pointed to a tarball, extract it to WORK_DIR if tarfile.is_tarfile(outfile) and file_xext(outfile).startswith('.tar'): @@ -1924,15 +1924,15 @@ def most_probable_alleles(allele_list): except KeyError: all_alleles[allele] = [float(pvalue)] # If there are less than 2 alleles, report all - if len(all_alleles.keys()) <= 2: - return all_alleles.keys() + if len(list(all_alleles.keys())) <= 2: + return list(all_alleles.keys()) # Else, get the two with most evidence. Evidence is gauged by # a) How many files (of the 3) thought that Allele was present # b) In a tie, who has a lower avg p value # In the lambda function, if 2 alleles have the same number of calls, the sum of the p values is # a measure of the avg because avg = sum / n and n is equal in both of them. else: - return sorted(all_alleles.keys(), key=lambda x: \ + return sorted(list(all_alleles.keys()), key=lambda x: \ (-len(all_alleles[x]), sum(all_alleles[x])))[0:2] @@ -2111,7 +2111,7 @@ def print_mhc_peptide(neoepitope_info, peptides, pepmap, outfile): """ allele, pept, pred, core = neoepitope_info - peptide_names = [x for x, y in peptides.items() if pept in y] + peptide_names = [x for x, y in list(peptides.items()) if pept in y] # For each peptide, append the ensembl gene for peptide_name in peptide_names: print(allele, pept, peptide_name, core, '0', pred, pepmap[peptide_name], sep='\t', @@ -2514,7 +2514,7 @@ def strip_xext(filepath): :return str filepath: Path to the file with the compression extension stripped off. """ ext_size = len(file_xext(filepath).split('.')) - 1 - for i in xrange(0, ext_size): + for i in range(0, ext_size): filepath = os.path.splitext(filepath)[0] return filepath diff --git a/attic/ProTECT_large.py b/attic/ProTECT_large.py index 40a7a200..485ea322 100644 --- a/attic/ProTECT_large.py +++ b/attic/ProTECT_large.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 Arjun Arkal Rao # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,7 @@ Program info can be found in the docstring of the main function. Details can also be obtained by running the script with -h . """ -from __future__ import print_function + import argparse import base64 @@ -79,7 +79,7 @@ def parse_config_file(job, config_file): # along with it's parameters. for groupname, group_params in tool_specific_param_generator(job, conf): if groupname == 'patient': - if 'patient_id' not in group_params.keys(): + if 'patient_id' not in list(group_params.keys()): raise ParameterError('A patient group is missing the patient_id flag.') sample_set[group_params['patient_id']] = group_params elif groupname == 'Universal_Options': @@ -101,7 +101,7 @@ def parse_config_file(job, config_file): raise ParameterError(' The following tools have no arguments in the config file : \n' + '\n'.join(missing_tools)) # Start a job for each sample in the sample set - for patient_id in sample_set.keys(): + for patient_id in list(sample_set.keys()): job.addFollowOnJobFn(pipeline_launchpad, sample_set[patient_id], univ_options, tool_options) return None @@ -304,7 +304,7 @@ def delete_fastqs(job, fastqs): +- 'normal_dna': [ , ] """ for fq_type in ['tumor_rna', 'tumor_dna', 'normal_dna']: - for i in xrange(0,2): + for i in range(0,2): job.fileStore.deleteGlobalFile(fastqs[fq_type][i]) return None @@ -708,7 +708,7 @@ def spawn_radia(job, rna_bam, tumor_bam, normal_bam, univ_options, radia_options 'normal_dnai': normal_bam['normal_dna_fix_pg_sorted.bam.bai']} # Make a dict object to hold the return values for each of the chromosome jobs. Then run radia # on each chromosome. - chromosomes = [''.join(['chr', str(x)]) for x in range(1, 23) + ['X', 'Y']] + chromosomes = [''.join(['chr', str(x)]) for x in list(range(1, 23)) + ['X', 'Y']] perchrom_radia = defaultdict() for chrom in chromosomes: perchrom_radia[chrom] = job.addChildJobFn(run_radia, bams, univ_options, radia_options, @@ -736,11 +736,11 @@ def merge_radia(job, perchrom_rvs): work_dir = job.fileStore.getLocalTempDir() # We need to squash the input dict of dicts to a single dict such that it can be passed to # get_files_from_filestore - input_files = {filename: jsid for perchrom_files in perchrom_rvs.values() - for filename, jsid in perchrom_files.items()} + input_files = {filename: jsid for perchrom_files in list(perchrom_rvs.values()) + for filename, jsid in list(perchrom_files.items())} input_files = get_files_from_filestore(job, input_files, work_dir, docker=False) - chromosomes = [''.join(['chr', str(x)]) for x in range(1, 23) + ['X', 'Y']] + chromosomes = [''.join(['chr', str(x)]) for x in list(range(1, 23)) + ['X', 'Y']] with open('/'.join([work_dir, 'radia_calls.vcf']), 'w') as radfile, \ open('/'.join([work_dir, 'radia_filter_passing_calls.vcf']), 'w') as radpassfile: for chrom in chromosomes: @@ -942,7 +942,7 @@ def spawn_mutect(job, tumor_bam, normal_bam, univ_options, mutect_options): job.fileStore.logToMaster('Running spawn_mutect on %s' % univ_options['patient']) # Make a dict object to hold the return values for each of the chromosome # jobs. Then run mutect on each chromosome. - chromosomes = [''.join(['chr', str(x)]) for x in range(1, 23) + ['X', 'Y']] + chromosomes = [''.join(['chr', str(x)]) for x in list(range(1, 23)) + ['X', 'Y']] perchrom_mutect = defaultdict() for chrom in chromosomes: perchrom_mutect[chrom] = job.addChildJobFn(run_mutect, tumor_bam, normal_bam, univ_options, @@ -968,10 +968,10 @@ def merge_mutect(job, perchrom_rvs): work_dir = job.fileStore.getLocalTempDir() # We need to squash the input dict of dicts to a single dict such that it can be passed to # get_files_from_filestore - input_files = {filename: jsid for perchrom_files in perchrom_rvs.values() - for filename, jsid in perchrom_files.items()} + input_files = {filename: jsid for perchrom_files in list(perchrom_rvs.values()) + for filename, jsid in list(perchrom_files.items())} input_files = get_files_from_filestore(job, input_files, work_dir, docker=False) - chromosomes = [''.join(['chr', str(x)]) for x in range(1, 23) + ['X', 'Y']] + chromosomes = [''.join(['chr', str(x)]) for x in list(range(1, 23)) + ['X', 'Y']] with open('/'.join([work_dir, 'mutect_calls.vcf']), 'w') as mutvcf, \ open('/'.join([work_dir, 'mutect_calls.out']), 'w') as mutout, \ open('/'.join([work_dir, 'mutect_passing_calls.vcf']), 'w') as mutpassvcf: @@ -1120,7 +1120,7 @@ def run_mutation_aggregator(job, fusion_output, radia_output, mutect_output, ind input_files.pop('fusion.vcf') # read files into memory vcf_file = defaultdict() - mutcallers = input_files.keys() + mutcallers = list(input_files.keys()) with open(''.join([work_dir, '/', univ_options['patient'], '_merged_mutations.vcf']), 'w') as merged_mut_file: for mut_caller in mutcallers: @@ -1566,8 +1566,8 @@ def merge_mhc_peptide_calls(job, antigen_predictions, transgened_files): mhci_files = get_files_from_filestore(job, mhci_preds, work_dir) # First split mhcii_preds into prediction files and predictors and maintain keys so we can later # reference them in pairs - mhcii_predictors = {x: y[1] for x, y in mhcii_preds.items()} - mhcii_files = {x: y[0] for x, y in mhcii_preds.items()} + mhcii_predictors = {x: y[1] for x, y in list(mhcii_preds.items())} + mhcii_files = {x: y[0] for x, y in list(mhcii_preds.items())} mhcii_files = get_files_from_filestore(job, mhcii_files, work_dir) # Get peptide files pept_files = get_files_from_filestore(job, pept_files, work_dir) @@ -1579,7 +1579,7 @@ def merge_mhc_peptide_calls(job, antigen_predictions, transgened_files): pepmap = json.load(mapfile) # Incorporate peptide names into the merged calls with open('/'.join([work_dir, 'mhci_merged_files.list']), 'w') as mhci_resfile: - for mhcifile in mhci_files.values(): + for mhcifile in list(mhci_files.values()): with open(mhcifile, 'r') as mf: for line in mf: # Skip header lines @@ -1600,7 +1600,7 @@ def merge_mhc_peptide_calls(job, antigen_predictions, transgened_files): # Incorporate peptide names into the merged calls with open('/'.join([work_dir, 'mhcii_merged_files.list']), 'w') as \ mhcii_resfile: - for mhciifile in mhcii_files.keys(): + for mhciifile in list(mhcii_files.keys()): core_col = None # Variable to hold the column number with the core if mhcii_predictors[mhciifile] == 'Consensus': with open(mhcii_files[mhciifile], 'r') as mf: @@ -1807,7 +1807,7 @@ def prepare_samples(job, fastqs, univ_options): 'normal_dna_fastq_prefix'} if set(fastqs.keys()).difference(allowed_samples) != {'patient_id'}: raise ParameterError('Sample with the following parameters has an error:\n' + - '\n'.join(fastqs.values())) + '\n'.join(list(fastqs.values()))) # For each sample type, check if the prefix is an S3 link or a regular file # Download S3 files. for sample_type in ['tumor_dna', 'tumor_rna', 'normal_dna']: @@ -1869,7 +1869,7 @@ def get_files_from_filestore(job, files, work_dir, cache=True, docker=False): work_dir is the location where the file should be stored cache indiciates whether caching should be used """ - for name in files.keys(): + for name in list(files.keys()): outfile = job.fileStore.readGlobalFile(files[name], '/'.join([work_dir, name]), cache=cache) # If the file pointed to a tarball, extract it to WORK_DIR if tarfile.is_tarfile(outfile) and file_xext(outfile).startswith('.tar'): @@ -1916,15 +1916,15 @@ def most_probable_alleles(allele_list): except KeyError: all_alleles[allele] = [float(pvalue)] # If there are less than 2 alleles, report all - if len(all_alleles.keys()) <= 2: - return all_alleles.keys() + if len(list(all_alleles.keys())) <= 2: + return list(all_alleles.keys()) # Else, get the two with most evidence. Evidence is gauged by # a) How many files (of the 3) thought that Allele was present # b) In a tie, who has a lower avg p value # In the lambda function, if 2 alleles have the same number of calls, the sum of the p values is # a measure of the avg because avg = sum / n and n is equal in both of them. else: - return sorted(all_alleles.keys(), key=lambda x: \ + return sorted(list(all_alleles.keys()), key=lambda x: \ (-len(all_alleles[x]), sum(all_alleles[x])))[0:2] @@ -2103,7 +2103,7 @@ def print_mhc_peptide(neoepitope_info, peptides, pepmap, outfile): """ allele, pept, pred, core = neoepitope_info - peptide_names = [x for x, y in peptides.items() if pept in y] + peptide_names = [x for x, y in list(peptides.items()) if pept in y] # For each peptide, append the ensembl gene for peptide_name in peptide_names: print(allele, pept, peptide_name, core, '0', pred, pepmap[peptide_name], sep='\t', @@ -2446,7 +2446,7 @@ def strip_xext(filepath): :return str filepath: Path to the file with the compression extension stripped off. """ ext_size = len(file_xext(filepath).split('.')) - 1 - for i in xrange(0, ext_size): + for i in range(0, ext_size): filepath = os.path.splitext(filepath)[0] return filepath diff --git a/attic/encrypt_files_in_dir_to_s3.py b/attic/encrypt_files_in_dir_to_s3.py index 67fac013..69bd0360 100644 --- a/attic/encrypt_files_in_dir_to_s3.py +++ b/attic/encrypt_files_in_dir_to_s3.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright (C) 2016 UCSC Computational Genomics Lab # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,7 +24,7 @@ Move files in a directory, or entire directory structures to S3 with (or without) encryption. """ -from __future__ import print_function + import argparse import base64 import hashlib diff --git a/attic/precision_immuno.py b/attic/precision_immuno.py index 73963b43..d958b8c2 100644 --- a/attic/precision_immuno.py +++ b/attic/precision_immuno.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 Arjun Arkal Rao # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,7 @@ Program info can be found in the docstring of the main function. Details can also be obtained by running the script with -h . """ -from __future__ import print_function + import argparse import base64 @@ -77,7 +77,7 @@ def parse_config_file(job, config_file): # along with it's parameters. for groupname, group_params in tool_specific_param_generator(job, conf): if groupname == 'patient': - if 'patient_id' not in group_params.keys(): + if 'patient_id' not in list(group_params.keys()): raise ParameterError('A patient group is missing the patient_id flag.') sample_set[group_params['patient_id']] = group_params elif groupname == 'Universal_Options': @@ -99,7 +99,7 @@ def parse_config_file(job, config_file): raise ParameterError(' The following tools have no arguments in the config file : \n' + '\n'.join(missing_tools)) # Start a job for each sample in the sample set - for patient_id in sample_set.keys(): + for patient_id in list(sample_set.keys()): job.addFollowOnJobFn(pipeline_launchpad, sample_set[patient_id], univ_options, tool_options) return None @@ -296,7 +296,7 @@ def delete_fastqs(job, fastqs): +- 'normal_dna': [ , ] """ for fq_type in ['tumor_rna', 'tumor_dna', 'normal_dna']: - for i in xrange(0,2): + for i in range(0,2): job.fileStore.deleteGlobalFile(fastqs[fq_type][i]) return None @@ -685,7 +685,7 @@ def spawn_radia(job, rna_bam, tumor_bam, normal_bam, univ_options, radia_options 'normal_dnai': normal_bam['normal_dna_fix_pg_sorted.bam.bai']} # Make a dict object to hold the return values for each of the chromosome jobs. Then run radia # on each chromosome. - chromosomes = [''.join(['chr', str(x)]) for x in range(1, 23) + ['X', 'Y']] + chromosomes = [''.join(['chr', str(x)]) for x in list(range(1, 23)) + ['X', 'Y']] perchrom_radia = defaultdict() for chrom in chromosomes: perchrom_radia[chrom] = job.addChildJobFn(run_radia, bams, univ_options, radia_options, @@ -710,11 +710,11 @@ def merge_radia(job, perchrom_rvs): work_dir = job.fileStore.getLocalTempDir() # We need to squash the input dict of dicts to a single dict such that it can be passed to # get_files_from_filestore - input_files = {filename: jsid for perchrom_files in perchrom_rvs.values() - for filename, jsid in perchrom_files.items()} + input_files = {filename: jsid for perchrom_files in list(perchrom_rvs.values()) + for filename, jsid in list(perchrom_files.items())} input_files = get_files_from_filestore(job, input_files, work_dir, docker=False) - chromosomes = [''.join(['chr', str(x)]) for x in range(1, 23) + ['X', 'Y']] + chromosomes = [''.join(['chr', str(x)]) for x in list(range(1, 23)) + ['X', 'Y']] with open('/'.join([work_dir, 'radia_calls.vcf']), 'w') as radfile, \ open('/'.join([work_dir, 'radia_filter_passing_calls.vcf']), 'w') as radpassfile: for chrom in chromosomes: @@ -909,7 +909,7 @@ def spawn_mutect(job, tumor_bam, normal_bam, univ_options, mutect_options): job.fileStore.logToMaster('Running spawn_mutect on %s' % univ_options['patient']) # Make a dict object to hold the return values for each of the chromosome # jobs. Then run mutect on each chromosome. - chromosomes = [''.join(['chr', str(x)]) for x in range(1, 23) + ['X', 'Y']] + chromosomes = [''.join(['chr', str(x)]) for x in list(range(1, 23)) + ['X', 'Y']] perchrom_mutect = defaultdict() for chrom in chromosomes: perchrom_mutect[chrom] = job.addChildJobFn(run_mutect, tumor_bam, normal_bam, univ_options, @@ -932,10 +932,10 @@ def merge_mutect(job, perchrom_rvs): work_dir = job.fileStore.getLocalTempDir() # We need to squash the input dict of dicts to a single dict such that it can be passed to # get_files_from_filestore - input_files = {filename: jsid for perchrom_files in perchrom_rvs.values() - for filename, jsid in perchrom_files.items()} + input_files = {filename: jsid for perchrom_files in list(perchrom_rvs.values()) + for filename, jsid in list(perchrom_files.items())} input_files = get_files_from_filestore(job, input_files, work_dir, docker=False) - chromosomes = [''.join(['chr', str(x)]) for x in range(1, 23) + ['X', 'Y']] + chromosomes = [''.join(['chr', str(x)]) for x in list(range(1, 23)) + ['X', 'Y']] with open('/'.join([work_dir, 'mutect_calls.vcf']), 'w') as mutvcf, \ open('/'.join([work_dir, 'mutect_calls.out']), 'w') as mutout, \ open('/'.join([work_dir, 'mutect_passing_calls.vcf']), 'w') as mutpassvcf: @@ -1076,7 +1076,7 @@ def run_mutation_aggregator(job, fusion_output, radia_output, mutect_output, ind input_files.pop('fusion.vcf') # read files into memory vcf_file = defaultdict() - mutcallers = input_files.keys() + mutcallers = list(input_files.keys()) with open(''.join([work_dir, '/', univ_options['patient'], '_merged_mutations.vcf']), 'w') as merged_mut_file: for mut_caller in mutcallers: @@ -1502,8 +1502,8 @@ def merge_mhc_peptide_calls(job, antigen_predictions, transgened_files): mhci_files = get_files_from_filestore(job, mhci_preds, work_dir) # First split mhcii_preds into prediction files and predictors and maintain keys so we can later # reference them in pairs - mhcii_predictors = {x: y[1] for x, y in mhcii_preds.items()} - mhcii_files = {x: y[0] for x, y in mhcii_preds.items()} + mhcii_predictors = {x: y[1] for x, y in list(mhcii_preds.items())} + mhcii_files = {x: y[0] for x, y in list(mhcii_preds.items())} mhcii_files = get_files_from_filestore(job, mhcii_files, work_dir) # Get peptide files pept_files = get_files_from_filestore(job, pept_files, work_dir) @@ -1515,7 +1515,7 @@ def merge_mhc_peptide_calls(job, antigen_predictions, transgened_files): pepmap = json.load(mapfile) # Incorporate peptide names into the merged calls with open('/'.join([work_dir, 'mhci_merged_files.list']), 'w') as mhci_resfile: - for mhcifile in mhci_files.values(): + for mhcifile in list(mhci_files.values()): with open(mhcifile, 'r') as mf: for line in mf: # Skip header lines @@ -1536,7 +1536,7 @@ def merge_mhc_peptide_calls(job, antigen_predictions, transgened_files): # Incorporate peptide names into the merged calls with open('/'.join([work_dir, 'mhcii_merged_files.list']), 'w') as \ mhcii_resfile: - for mhciifile in mhcii_files.keys(): + for mhciifile in list(mhcii_files.keys()): core_col = None # Variable to hold the column number with the core if mhcii_predictors[mhciifile] == 'Consensus': with open(mhcii_files[mhciifile], 'r') as mf: @@ -1740,7 +1740,7 @@ def prepare_samples(job, fastqs, univ_options): 'normal_dna_fastq_prefix'} if set(fastqs.keys()).difference(allowed_samples) != {'patient_id'}: raise ParameterError('Sample with the following parameters has an error:\n' + - '\n'.join(fastqs.values())) + '\n'.join(list(fastqs.values()))) # For each sample type, check if the prefix is an S3 link or a regular file # Download S3 files. for sample_type in ['tumor_dna', 'tumor_rna', 'normal_dna']: @@ -1800,7 +1800,7 @@ def get_files_from_filestore(job, files, work_dir, cache=True, docker=False): work_dir is the location where the file should be stored cache indiciates whether caching should be used """ - for name in files.keys(): + for name in list(files.keys()): outfile = job.fileStore.readGlobalFile(files[name], '/'.join([work_dir, name]), cache=cache) # If the file pointed to a tarball, extract it to WORK_DIR if tarfile.is_tarfile(outfile) and file_xext(outfile).startswith('.tar'): @@ -1847,15 +1847,15 @@ def most_probable_alleles(allele_list): except KeyError: all_alleles[allele] = [float(pvalue)] # If there are less than 2 alleles, report all - if len(all_alleles.keys()) <= 2: - return all_alleles.keys() + if len(list(all_alleles.keys())) <= 2: + return list(all_alleles.keys()) # Else, get the two with most evidence. Evidence is gauged by # a) How many files (of the 3) thought that Allele was present # b) In a tie, who has a lower avg p value # In the lambda function, if 2 alleles have the same number of calls, the sum of the p values is # a measure of the avg because avg = sum / n and n is equal in both of them. else: - return sorted(all_alleles.keys(), key=lambda x: \ + return sorted(list(all_alleles.keys()), key=lambda x: \ (-len(all_alleles[x]), sum(all_alleles[x])))[0:2] @@ -2031,7 +2031,7 @@ def print_mhc_peptide(neoepitope_info, peptides, pepmap, outfile): 'ensembl_gene\thugo_gene\tcomma_sep_transcript_mutations' """ allele, pept, pred, core = neoepitope_info - peptide_names = [x for x, y in peptides.items() if pept in y] + peptide_names = [x for x, y in list(peptides.items()) if pept in y] # For each peptide, append the ensembl gene for peptide_name in peptide_names: print(allele, pept, peptide_name, core, '0', pred, pepmap[peptide_name], sep='\t', @@ -2368,7 +2368,7 @@ def strip_xext(filepath): :return str filepath: Path to the file with the compression extension stripped off. """ ext_size = len(file_xext(filepath).split('.')) - 1 - for i in xrange(0, ext_size): + for i in range(0, ext_size): filepath = os.path.splitext(filepath)[0] return filepath diff --git a/changes.md b/changes.md new file mode 100644 index 00000000..1d7fa69c --- /dev/null +++ b/changes.md @@ -0,0 +1,22 @@ +# Manual Changes to ProTECT +:star: indicates changes to the algorithm that *could* potentially change results, though best efforts were made for 1:1 conversion + +:black_square_button: indicates changes i hope to reverse and are only 'temp fixes' + +- Originally ran 2to3 in commit a5d062fab68f8bbbebc2bbe9f4192b47b451146e + - removed explicit versioning in the [Makefile](https://github.com/BD2KGenomics/protect/commit/a5d062fab68f8bbbebc2bbe9f4192b47b451146e#diff-76ed074a9305c04054cdebb9e9aad2d818052b07091de1f20cad0bbac34ffb52) since py3 version still in dev +- :black_square_button: [removed version checks in setup.py](https://github.com/BD2KGenomics/protect/commit/f04f22fb9f50270e5c0307d4a64aca0f3f7022d3) and obsolete setuptools + - [along with setup version](https://github.com/BD2KGenomics/protect/commit/f70d3196198a2530406906b8af5a55b848aa0b14) +- [changed default references](https://github.com/BD2KGenomics/protect/commit/c2fe3a8b8223682e6d63cccb4fccf0787227c525) from s3://cgl-pipeline-inputs to s3://protect-data + - this s3 bucket is pay to access, however currently s3am is untested and only automatically converted +- :star::star: [common.py chromosome sorting](https://github.com/BD2KGenomics/protect/commit/b5ca956f3dfe05bf6714be8135cc90fe48140d98) +- [docker image decodes to utf-8](https://github.com/BD2KGenomics/protect/commit/1d2bdb941548bdf4703113140d1f0758791bf88a) +- [IOBase check rather than file](https://github.com/BD2KGenomics/protect/commit/351e855184ae218242a42e1aaa5781d22aba0511) +- [some binary vs string adaptations](https://github.com/BD2KGenomics/protect/commit/5a4c50d1d2b8c71f3bc2f512f3679e80368044be#diff-e46b0e6e9cc33d9130334ab6994c9684b0972aaca58c889b6c1f4819751f1c79) +- :star: [change obselete ix to loc](https://github.com/BD2KGenomics/protect/commit/66bc12db0b815ab2099ee0174a06b923240322a4#diff-3347ae223ced4e929cf7f273bf839bdeb219d82681f8e66a951d85cbeb079685) +- :black_square_button: quick fix: running into problem with '80.0' default for cores. can't figure out where the default is coming from so manual changes: + - [phlat cores to 10](https://github.com/BD2KGenomics/protect/commit/66bc12db0b815ab2099ee0174a06b923240322a4#diff-7e85a3e4e9c911fded129ff48b2dd983d800c5190412f641eee85ff23ed9295c) + - [rsem cores to 20](https://github.com/BD2KGenomics/protect/commit/66bc12db0b815ab2099ee0174a06b923240322a4#diff-1615337ffdbffe39413f26e4ccbb5309ed10b61987559df23ce6fc57cb5dd86a) + - [star fusion cores to 20](https://github.com/BD2KGenomics/protect/commit/66bc12db0b815ab2099ee0174a06b923240322a4#diff-60e2cfd2feabfe71442d69d1d0d44ff293f8fe1e12aa74c3fe52101d5b32e60e) +- [string.maketrans is obselete, str.maketrans is better](https://github.com/BD2KGenomics/protect/commit/66bc12db0b815ab2099ee0174a06b923240322a4#diff-60e2cfd2feabfe71442d69d1d0d44ff293f8fe1e12aa74c3fe52101d5b32e60eR300) +- changed gunzip file write to use a library (faster?) diff --git a/docker/pipelineWrapper.py b/docker/pipelineWrapper.py index d767e41b..12502809 100644 --- a/docker/pipelineWrapper.py +++ b/docker/pipelineWrapper.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function + import tarfile import argparse @@ -103,7 +103,7 @@ def getName(fileName): # move individual files out for fileName in consistentNaming: shutil.copyfile(getName(fileName), os.path.join(output_dir, os.path.basename(fileName))) - for src, dst in renamingNeeded.iteritems(): + for src, dst in renamingNeeded.items(): if dst.endswith('.tar'): make_tar(getName(src), os.path.join(output_dir, dst)) else: diff --git a/docker/wrapper.py b/docker/wrapper.py index c142f387..0d08c223 100644 --- a/docker/wrapper.py +++ b/docker/wrapper.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function + from pipelineWrapper import PipelineWrapperBuilder import logging import os @@ -437,20 +437,20 @@ def str2bool(v): help='Tabix index for dbsnp.gz.') parser.add_argument('--mhc-pathways-file', type=str, - default="S3://cgl-pipeline-inputs/protect/ci_references/" + default="S3://protect-data/hg38_references/" "mhc_pathways.tsv.tar.gz", help='JSON file containing the various genes in the MHC pathway' 'and their mean TPM expressions across samples in a background set.') parser.add_argument('--itx-resistance-file', type=str, - default="S3://cgl-pipeline-inputs/protect/ci_references/" + default="S3://protect-data/hg38_references/" "itx_resistance.tsv.tar.gz", help='') parser.add_argument('--immune-resistance-pathways-file', type=str, - default="S3://cgl-pipeline-inputs/protect/ci_references/" + default="S3://protect-data/hg38_references/" "immune_resistance_pathways.json.tar.gz", help='') parser.add_argument('--car-t-targets-file', type=str, - default="S3://cgl-pipeline-inputs/protect/ci_references/" + default="S3://protect-data/hg38_references/" "car_t_targets.tsv.tar.gzz", help='') diff --git a/errors/2020-10-19-14-57-49.txt b/errors/2020-10-19-14-57-49.txt new file mode 100644 index 00000000..3eda3965 --- /dev/null +++ b/errors/2020-10-19-14-57-49.txt @@ -0,0 +1,438 @@ +[2020-10-19T14:57:49-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-foexdy0m +[2020-10-19T14:57:49-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. +[2020-10-19T14:57:49-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-foexdy0m with job batch system ID: 0 and cores: 4, disk: 7.8 G, and memory: 2.0 G +[2020-10-19T14:57:49-0700] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-n45358fz with job batch system ID: 1 and cores: 4, disk: 9.6 G, and memory: 12.0 G +[2020-10-19T14:57:49-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-xpy0rfpy with job batch system ID: 2 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-19T14:57:49-0700] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-75n4628j with job batch system ID: 3 and cores: 4, disk: 9.4 G, and memory: 12.0 G +[2020-10-19T14:57:49-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-2oiyd0li with job batch system ID: 4 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-19T14:57:49-0700] [MainThread] [I] [toil.leader] Issued job 'run_star' kind-run_star/instance-rpbnfzyy with job batch system ID: 5 and cores: 4, disk: 52.8 G, and memory: 48.6 G +[2020-10-19T14:57:49-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpj8pqddja/worker_log.txt +[2020-10-19T14:57:51-0700] [MainThread] [I] [toil.leader] 1 jobs are running, 5 jobs are issued and waiting to run +[2020-10-19T15:00:33-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/home/drkthomp/protect3/jobStore kind-run_phlat/instance-foexdy0m. +[2020-10-19T15:00:33-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-foexdy0m +[2020-10-19T15:00:33-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-foexdy0m +[2020-10-19T15:00:33-0700] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-foexdy0m follows: +=========> + [2020-10-19T14:57:49-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T14:57:49-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + 567573 reads; of these: + 567573 (100.00%) were paired; of these: + 3570 (0.63%) aligned concordantly 0 times + 514680 (90.68%) aligned concordantly exactly 1 time + 49323 (8.69%) aligned concordantly >1 times + 99.37% overall alignment rate + .....Process Bowtie 2 mapping on tumor_dna....... + + .....Prepare files of tumor_dna for PHLAT....... + + ..... Running PHLAT ....... + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 64, in + go(tag,outdir,phlatdir) + File "/home/yubai/programs/phlat-1.0/dist/utilities.py", line 79, in go + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 617, in __init__ + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 645, in setMaster + IndexError: list index out of range + Traceback (most recent call last): + File "/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib/python3.8/subprocess.py", line 364, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpj8pqddja/a0ac95e2-e630-4058-8af8-896d4f133203:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '4']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 73, in run_phlat + docker_call(tool='phlat', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 138, in docker_call + raise RuntimeError('docker command returned a non-zero exit status (%s)' % err.returncode + + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpj8pqddja/a0ac95e2-e630-4058-8af8-896d4f133203:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_dna -e /home/phlat-1.0 -o /data -p 4" + [2020-10-19T15:00:33-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox + {'input_1.fastq': '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpj8pqddja/a0ac95e2-e630-4058-8af8-896d4f133203/input_1.fastq', 'input_2.fastq': '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpj8pqddja/a0ac95e2-e630-4058-8af8-896d4f133203/input_2.fastq', 'phlat_index.tar.gz': '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpj8pqddja/a0ac95e2-e630-4058-8af8-896d4f133203/phlat_index.tar.gz'} +<========= +[2020-10-19T15:00:33-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-foexdy0m with ID kind-run_phlat/instance-foexdy0m to 1 +[2020-10-19T15:00:33-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-foexdy0m with job batch system ID: 6 and cores: 4, disk: 7.8 G, and memory: 2.0 G +[2020-10-19T15:00:35-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpuh28jgni/worker_log.txt +[2020-10-19T15:03:06-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/home/drkthomp/protect3/jobStore kind-run_phlat/instance-xpy0rfpy. +[2020-10-19T15:03:06-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-xpy0rfpy +[2020-10-19T15:03:06-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-xpy0rfpy +[2020-10-19T15:03:06-0700] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-xpy0rfpy follows: +=========> + [2020-10-19T15:00:35-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T15:00:35-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + 491247 reads; of these: + 491247 (100.00%) were paired; of these: + 3488 (0.71%) aligned concordantly 0 times + 442898 (90.16%) aligned concordantly exactly 1 time + 44861 (9.13%) aligned concordantly >1 times + 99.29% overall alignment rate + .....Process Bowtie 2 mapping on normal_dna....... + + .....Prepare files of normal_dna for PHLAT....... + + ..... Running PHLAT ....... + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 64, in + go(tag,outdir,phlatdir) + File "/home/yubai/programs/phlat-1.0/dist/utilities.py", line 79, in go + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 617, in __init__ + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 645, in setMaster + IndexError: list index out of range + Traceback (most recent call last): + File "/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib/python3.8/subprocess.py", line 364, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpuh28jgni/5320bc4c-c2cd-451c-b94e-490c3ec13e8f:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'normal_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '4']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 73, in run_phlat + docker_call(tool='phlat', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 138, in docker_call + raise RuntimeError('docker command returned a non-zero exit status (%s)' % err.returncode + + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpuh28jgni/5320bc4c-c2cd-451c-b94e-490c3ec13e8f:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag normal_dna -e /home/phlat-1.0 -o /data -p 4" + [2020-10-19T15:03:06-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox + {'input_1.fastq': '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpuh28jgni/5320bc4c-c2cd-451c-b94e-490c3ec13e8f/input_1.fastq', 'input_2.fastq': '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpuh28jgni/5320bc4c-c2cd-451c-b94e-490c3ec13e8f/input_2.fastq', 'phlat_index.tar.gz': '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpuh28jgni/5320bc4c-c2cd-451c-b94e-490c3ec13e8f/phlat_index.tar.gz'} +<========= +[2020-10-19T15:03:06-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-xpy0rfpy with ID kind-run_phlat/instance-xpy0rfpy to 1 +[2020-10-19T15:03:06-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-xpy0rfpy with job batch system ID: 7 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-19T15:03:07-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmps5g5jnhi/worker_log.txt +[2020-10-19T15:04:32-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_bwa file:/home/drkthomp/protect3/jobStore kind-run_bwa/instance-n45358fz. +[2020-10-19T15:04:32-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_bwa' kind-run_bwa/instance-n45358fz +[2020-10-19T15:04:32-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_bwa' kind-run_bwa/instance-n45358fz +[2020-10-19T15:04:32-0700] [MainThread] [W] [toil.leader] Log from job kind-run_bwa/instance-n45358fz follows: +=========> + [2020-10-19T15:03:07-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T15:03:07-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/alignment/dna.py", line 140, in run_bwa + docker_call(tool='bwa', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 101, in docker_call + assert isinstance(outfile, file), 'outfile was not passsed a file' + NameError: name 'file' is not defined + [2020-10-19T15:04:31-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-19T15:04:32-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_bwa' kind-run_bwa/instance-n45358fz with ID kind-run_bwa/instance-n45358fz to 1 +[2020-10-19T15:04:32-0700] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-n45358fz with job batch system ID: 8 and cores: 4, disk: 9.6 G, and memory: 12.0 G +[2020-10-19T15:04:32-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpx9ev25qk/worker_log.txt +[2020-10-19T15:07:09-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/home/drkthomp/protect3/jobStore kind-run_phlat/instance-xpy0rfpy. +[2020-10-19T15:07:09-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-xpy0rfpy +[2020-10-19T15:07:09-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-xpy0rfpy +[2020-10-19T15:07:09-0700] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-xpy0rfpy follows: +=========> + [2020-10-19T15:04:32-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T15:04:32-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + 491247 reads; of these: + 491247 (100.00%) were paired; of these: + 3488 (0.71%) aligned concordantly 0 times + 442898 (90.16%) aligned concordantly exactly 1 time + 44861 (9.13%) aligned concordantly >1 times + 99.29% overall alignment rate + .....Process Bowtie 2 mapping on normal_dna....... + + .....Prepare files of normal_dna for PHLAT....... + + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 64, in + ..... Running PHLAT ....... + go(tag,outdir,phlatdir) + File "/home/yubai/programs/phlat-1.0/dist/utilities.py", line 79, in go + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 617, in __init__ + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 645, in setMaster + IndexError: list index out of range + Traceback (most recent call last): + File "/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib/python3.8/subprocess.py", line 364, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpx9ev25qk/f858a60a-81a6-4c08-9472-6a0085cc69ea:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'normal_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '4']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 73, in run_phlat + docker_call(tool='phlat', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 138, in docker_call + raise RuntimeError('docker command returned a non-zero exit status (%s)' % err.returncode + + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpx9ev25qk/f858a60a-81a6-4c08-9472-6a0085cc69ea:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag normal_dna -e /home/phlat-1.0 -o /data -p 4" + [2020-10-19T15:07:09-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox + {'input_1.fastq': '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpx9ev25qk/f858a60a-81a6-4c08-9472-6a0085cc69ea/input_1.fastq', 'input_2.fastq': '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpx9ev25qk/f858a60a-81a6-4c08-9472-6a0085cc69ea/input_2.fastq', 'phlat_index.tar.gz': '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpx9ev25qk/f858a60a-81a6-4c08-9472-6a0085cc69ea/phlat_index.tar.gz'} +<========= +[2020-10-19T15:07:09-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-xpy0rfpy with ID kind-run_phlat/instance-xpy0rfpy to 0 +[2020-10-19T15:07:09-0700] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-xpy0rfpy with ID kind-run_phlat/instance-xpy0rfpy is completely failed +[2020-10-19T15:07:10-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpg3b7n5f7/worker_log.txt +[2020-10-19T15:08:06-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_bwa file:/home/drkthomp/protect3/jobStore kind-run_bwa/instance-n45358fz. +[2020-10-19T15:08:06-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_bwa' kind-run_bwa/instance-n45358fz +[2020-10-19T15:08:06-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_bwa' kind-run_bwa/instance-n45358fz +[2020-10-19T15:08:06-0700] [MainThread] [W] [toil.leader] Log from job kind-run_bwa/instance-n45358fz follows: +=========> + [2020-10-19T15:07:10-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T15:07:10-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/alignment/dna.py", line 140, in run_bwa + docker_call(tool='bwa', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 101, in docker_call + assert isinstance(outfile, file), 'outfile was not passsed a file' + NameError: name 'file' is not defined + [2020-10-19T15:08:04-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-19T15:08:06-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_bwa' kind-run_bwa/instance-n45358fz with ID kind-run_bwa/instance-n45358fz to 0 +[2020-10-19T15:08:06-0700] [MainThread] [W] [toil.leader] Job 'run_bwa' kind-run_bwa/instance-n45358fz with ID kind-run_bwa/instance-n45358fz is completely failed +[2020-10-19T15:08:06-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmp67xxzuvv/worker_log.txt +[2020-10-19T15:11:07-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/home/drkthomp/protect3/jobStore kind-run_phlat/instance-foexdy0m. +[2020-10-19T15:11:07-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-foexdy0m +[2020-10-19T15:11:07-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-foexdy0m +[2020-10-19T15:11:07-0700] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-foexdy0m follows: +=========> + [2020-10-19T15:08:06-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T15:08:06-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + 567573 reads; of these: + 567573 (100.00%) were paired; of these: + 3570 (0.63%) aligned concordantly 0 times + 514680 (90.68%) aligned concordantly exactly 1 time + 49323 (8.69%) aligned concordantly >1 times + 99.37% overall alignment rate + .....Process Bowtie 2 mapping on tumor_dna....... + + .....Prepare files of tumor_dna for PHLAT....... + + ..... Running PHLAT ....... + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 64, in + go(tag,outdir,phlatdir) + File "/home/yubai/programs/phlat-1.0/dist/utilities.py", line 79, in go + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 617, in __init__ + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 645, in setMaster + IndexError: list index out of range + Traceback (most recent call last): + File "/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib/python3.8/subprocess.py", line 364, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmp67xxzuvv/78e94666-8f7d-4d01-b17f-0e201d55b8bb:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '4']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 73, in run_phlat + docker_call(tool='phlat', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 138, in docker_call + raise RuntimeError('docker command returned a non-zero exit status (%s)' % err.returncode + + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmp67xxzuvv/78e94666-8f7d-4d01-b17f-0e201d55b8bb:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_dna -e /home/phlat-1.0 -o /data -p 4" + [2020-10-19T15:11:07-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox + {'input_1.fastq': '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmp67xxzuvv/78e94666-8f7d-4d01-b17f-0e201d55b8bb/input_1.fastq', 'input_2.fastq': '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmp67xxzuvv/78e94666-8f7d-4d01-b17f-0e201d55b8bb/input_2.fastq', 'phlat_index.tar.gz': '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmp67xxzuvv/78e94666-8f7d-4d01-b17f-0e201d55b8bb/phlat_index.tar.gz'} +<========= +[2020-10-19T15:11:07-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-foexdy0m with ID kind-run_phlat/instance-foexdy0m to 0 +[2020-10-19T15:11:07-0700] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-foexdy0m with ID kind-run_phlat/instance-foexdy0m is completely failed +[2020-10-19T15:11:08-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmph10zw7zc/worker_log.txt +[2020-10-19T15:12:03-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_bwa file:/home/drkthomp/protect3/jobStore kind-run_bwa/instance-75n4628j. +[2020-10-19T15:12:03-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_bwa' kind-run_bwa/instance-75n4628j +[2020-10-19T15:12:03-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_bwa' kind-run_bwa/instance-75n4628j +[2020-10-19T15:12:03-0700] [MainThread] [W] [toil.leader] Log from job kind-run_bwa/instance-75n4628j follows: +=========> + [2020-10-19T15:11:08-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T15:11:08-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/alignment/dna.py", line 140, in run_bwa + docker_call(tool='bwa', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 101, in docker_call + assert isinstance(outfile, file), 'outfile was not passsed a file' + NameError: name 'file' is not defined + [2020-10-19T15:12:01-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-19T15:12:03-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_bwa' kind-run_bwa/instance-75n4628j with ID kind-run_bwa/instance-75n4628j to 1 +[2020-10-19T15:12:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-75n4628j with job batch system ID: 9 and cores: 4, disk: 9.4 G, and memory: 12.0 G +[2020-10-19T15:12:03-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpl2u2480c/worker_log.txt +[2020-10-19T15:14:18-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/home/drkthomp/protect3/jobStore kind-run_phlat/instance-2oiyd0li. +[2020-10-19T15:14:18-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-2oiyd0li +[2020-10-19T15:14:18-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-2oiyd0li +[2020-10-19T15:14:18-0700] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-2oiyd0li follows: +=========> + [2020-10-19T15:12:03-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T15:12:03-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + 462668 reads; of these: + 462668 (100.00%) were paired; of these: + 75207 (16.26%) aligned concordantly 0 times + 235430 (50.89%) aligned concordantly exactly 1 time + 152031 (32.86%) aligned concordantly >1 times + 83.74% overall alignment rate + .....Process Bowtie 2 mapping on tumor_rna....... + + .....Prepare files of tumor_rna for PHLAT....... + + ..... Running PHLAT ....... + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 64, in + go(tag,outdir,phlatdir) + File "/home/yubai/programs/phlat-1.0/dist/utilities.py", line 79, in go + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 617, in __init__ + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 645, in setMaster + IndexError: list index out of range + Traceback (most recent call last): + File "/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib/python3.8/subprocess.py", line 364, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpl2u2480c/3f892483-6778-4874-af82-231b605ff625:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_rna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '4']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 73, in run_phlat + docker_call(tool='phlat', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 138, in docker_call + raise RuntimeError('docker command returned a non-zero exit status (%s)' % err.returncode + + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpl2u2480c/3f892483-6778-4874-af82-231b605ff625:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_rna -e /home/phlat-1.0 -o /data -p 4" + [2020-10-19T15:14:18-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox + {'input_1.fastq': '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpl2u2480c/3f892483-6778-4874-af82-231b605ff625/input_1.fastq', 'input_2.fastq': '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpl2u2480c/3f892483-6778-4874-af82-231b605ff625/input_2.fastq', 'phlat_index.tar.gz': '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpl2u2480c/3f892483-6778-4874-af82-231b605ff625/phlat_index.tar.gz'} +<========= +[2020-10-19T15:14:18-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-2oiyd0li with ID kind-run_phlat/instance-2oiyd0li to 1 +[2020-10-19T15:14:18-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-2oiyd0li with job batch system ID: 10 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-19T15:14:19-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpydrl0r5w/worker_log.txt +[2020-10-19T15:15:17-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_bwa file:/home/drkthomp/protect3/jobStore kind-run_bwa/instance-75n4628j. +[2020-10-19T15:15:17-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_bwa' kind-run_bwa/instance-75n4628j +[2020-10-19T15:15:17-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_bwa' kind-run_bwa/instance-75n4628j +[2020-10-19T15:15:17-0700] [MainThread] [W] [toil.leader] Log from job kind-run_bwa/instance-75n4628j follows: +=========> + [2020-10-19T15:14:19-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T15:14:19-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/alignment/dna.py", line 140, in run_bwa + docker_call(tool='bwa', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 101, in docker_call + assert isinstance(outfile, file), 'outfile was not passsed a file' + NameError: name 'file' is not defined + [2020-10-19T15:15:16-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-19T15:15:17-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_bwa' kind-run_bwa/instance-75n4628j with ID kind-run_bwa/instance-75n4628j to 0 +[2020-10-19T15:15:17-0700] [MainThread] [W] [toil.leader] Job 'run_bwa' kind-run_bwa/instance-75n4628j with ID kind-run_bwa/instance-75n4628j is completely failed +[2020-10-19T15:15:18-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpyqubeh1w/worker_log.txt +[2020-10-19T15:17:37-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/home/drkthomp/protect3/jobStore kind-run_phlat/instance-2oiyd0li. +[2020-10-19T15:17:37-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-2oiyd0li +[2020-10-19T15:17:37-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-2oiyd0li +[2020-10-19T15:17:37-0700] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-2oiyd0li follows: +=========> + [2020-10-19T15:15:18-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T15:15:18-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + 462668 reads; of these: + 462668 (100.00%) were paired; of these: + 75207 (16.26%) aligned concordantly 0 times + 235430 (50.89%) aligned concordantly exactly 1 time + 152031 (32.86%) aligned concordantly >1 times + 83.74% overall alignment rate + .....Process Bowtie 2 mapping on tumor_rna....... + + .....Prepare files of tumor_rna for PHLAT....... + + ..... Running PHLAT ....... + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 64, in + go(tag,outdir,phlatdir) + File "/home/yubai/programs/phlat-1.0/dist/utilities.py", line 79, in go + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 617, in __init__ + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 645, in setMaster + IndexError: list index out of range + Traceback (most recent call last): + File "/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib/python3.8/subprocess.py", line 364, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpyqubeh1w/b2b182ee-f0fa-4d4e-9fc7-ba1a0a3a14e9:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_rna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '4']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 73, in run_phlat + docker_call(tool='phlat', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 138, in docker_call + raise RuntimeError('docker command returned a non-zero exit status (%s)' % err.returncode + + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpyqubeh1w/b2b182ee-f0fa-4d4e-9fc7-ba1a0a3a14e9:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_rna -e /home/phlat-1.0 -o /data -p 4" + [2020-10-19T15:17:36-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox + {'input_1.fastq': '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpyqubeh1w/b2b182ee-f0fa-4d4e-9fc7-ba1a0a3a14e9/input_1.fastq', 'input_2.fastq': '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpyqubeh1w/b2b182ee-f0fa-4d4e-9fc7-ba1a0a3a14e9/input_2.fastq', 'phlat_index.tar.gz': '/home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmpyqubeh1w/b2b182ee-f0fa-4d4e-9fc7-ba1a0a3a14e9/phlat_index.tar.gz'} +<========= +[2020-10-19T15:17:37-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-2oiyd0li with ID kind-run_phlat/instance-2oiyd0li to 0 +[2020-10-19T15:17:37-0700] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-2oiyd0li with ID kind-run_phlat/instance-2oiyd0li is completely failed +[2020-10-19T15:57:52-0700] [MainThread] [I] [toil.leader] 0 jobs are running, 1 jobs are issued and waiting to run diff --git a/errors/2020-10-19-14.txt b/errors/2020-10-19-14.txt new file mode 100644 index 00000000..17e88987 --- /dev/null +++ b/errors/2020-10-19-14.txt @@ -0,0 +1,10 @@ +[2020-10-19T14:55:25-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-foexdy0m +[2020-10-19T14:55:25-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. +[2020-10-19T14:55:25-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-foexdy0m with job batch system ID: 0 and cores: 4, disk: 7.8 G, and memory: 2.0 G +[2020-10-19T14:55:25-0700] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-n45358fz with job batch system ID: 1 and cores: 4, disk: 9.6 G, and memory: 12.0 G +[2020-10-19T14:55:25-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-xpy0rfpy with job batch system ID: 2 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-19T14:55:25-0700] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-75n4628j with job batch system ID: 3 and cores: 4, disk: 9.4 G, and memory: 12.0 G +[2020-10-19T14:55:25-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-2oiyd0li with job batch system ID: 4 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-19T14:55:25-0700] [MainThread] [I] [toil.leader] Issued job 'run_star' kind-run_star/instance-rpbnfzyy with job batch system ID: 5 and cores: 4, disk: 52.8 G, and memory: 48.6 G +[2020-10-19T14:55:25-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-77a2b3ac-69eb-4325-9205-afc132e2b6e3-1e53f69f16d74e0b881e7db602b2d2c3/tmplgo6vvm6/worker_log.txt +[2020-10-19T14:55:27-0700] [MainThread] [I] [toil.leader] 1 jobs are running, 5 jobs are issued and waiting to run diff --git a/errors/2020-10-19-16-54-58.txt b/errors/2020-10-19-16-54-58.txt new file mode 100644 index 00000000..e1aa3802 --- /dev/null +++ b/errors/2020-10-19-16-54-58.txt @@ -0,0 +1,12 @@ +Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/bin/ProTECT", line 33, in + sys.exit(load_entry_point('protect', 'console_scripts', 'ProTECT')()) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1025, in main + Job.Runner.startToil(start, params) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 794, in startToil + with Toil(options) as toil: + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/common.py", line 776, in __enter__ + jobStore.resume() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/jobStores/fileJobStore.py", line 121, in resume + raise NoSuchJobStoreException(self.jobStoreDir) +toil.jobStores.abstractJobStore.NoSuchJobStoreException: The job store '/home/drkthomp/protect3/jobStore' does not exist, so there is nothing to restart. diff --git a/errors/2020-10-19-16-55-14.txt b/errors/2020-10-19-16-55-14.txt new file mode 100644 index 00000000..5e5dea8d --- /dev/null +++ b/errors/2020-10-19-16-55-14.txt @@ -0,0 +1,613 @@ +[2020-10-19T16:55:14-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. +[2020-10-19T16:55:14-0700] [MainThread] [I] [toil.leader] Issued job 'parse_config_file' kind-parse_config_file/instance-6z_yzhbw with job batch system ID: 0 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:14-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpzwunuav1/worker_log.txt +[2020-10-19T16:55:15-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 16:55:15: Parsing config file +[2020-10-19T16:55:15-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 16:55:15: Obtaining tool inputs +[2020-10-19T16:55:15-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 16:55:15: Obtained tool inputs +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Job ended: 'parse_config_file' kind-parse_config_file/instance-6z_yzhbw +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] 0 jobs are running, 0 jobs are issued and waiting to run +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-3ot0nqrl with job batch system ID: 1 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-zq7r2fof with job batch system ID: 2 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-jzbemfd2 with job batch system ID: 3 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-a82w2061 with job batch system ID: 4 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-sohjmdrx with job batch system ID: 5 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-s7uqh4l5 with job batch system ID: 6 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-wsb3vfxw with job batch system ID: 7 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-vz7a41kb with job batch system ID: 8 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ue7dbx_y with job batch system ID: 9 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-b2p8fmcs with job batch system ID: 10 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-7aptjv8m with job batch system ID: 11 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-88k469b1 with job batch system ID: 12 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-8nav8g_w with job batch system ID: 13 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1nv69jcb with job batch system ID: 14 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1ibo1747 with job batch system ID: 15 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-2pfe72sp with job batch system ID: 16 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-hbd_web4 with job batch system ID: 17 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-vfbtut8x with job batch system ID: 18 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-qr4sdt3x with job batch system ID: 19 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6s511qxp with job batch system ID: 20 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-zf_4y0r9 with job batch system ID: 21 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-z1kq216o with job batch system ID: 22 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-wu4un7pj with job batch system ID: 23 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-qeqilzdt with job batch system ID: 24 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-_tjtgbal with job batch system ID: 25 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-_vdkxdam with job batch system ID: 26 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-38htzb_g with job batch system ID: 27 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-4q9y9qdt with job batch system ID: 28 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-affo6juu with job batch system ID: 29 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpyabe_f4v/worker_log.txt +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpmak877i1/worker_log.txt +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpg38fsy_q/worker_log.txt +[2020-10-19T16:55:15-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpyydetbh9/worker_log.txt +[2020-10-19T17:05:14-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-a82w2061 +[2020-10-19T17:05:14-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:05:14: Obtaining file (indexes:genome_fasta) to the file job store +[2020-10-19T17:05:14-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpvy2fjfbk/worker_log.txt +[2020-10-19T17:05:15-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-vz7a41kb +[2020-10-19T17:05:15-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:05:15: Obtaining file (indexes:cosmic_idx) to the file job store +[2020-10-19T17:05:15-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp7ifahyb5/worker_log.txt +[2020-10-19T17:05:16-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-4q9y9qdt +[2020-10-19T17:05:16-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:05:16: Obtaining file (reports:immune_resistance_pathways_file) to the file job store +[2020-10-19T17:05:16-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmphcj22yyc/worker_log.txt +[2020-10-19T17:06:51-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-jzbemfd2 +[2020-10-19T17:06:51-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpb5s_mwjh/worker_log.txt +[2020-10-19T17:06:51-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:06:51: Obtaining file (rsem:index) to the file job store +[2020-10-19T17:07:26-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-wsb3vfxw +[2020-10-19T17:07:26-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:07:26: Obtaining file (indexes:cosmic_vcf) to the file job store +[2020-10-19T17:07:27-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpxzkxoeiv/worker_log.txt +[2020-10-19T17:07:27-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-38htzb_g +[2020-10-19T17:07:27-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:07:27: Obtaining file (reports:itx_resistance_file) to the file job store +[2020-10-19T17:07:27-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpydeo3m77/worker_log.txt +[2020-10-19T17:07:34-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-qr4sdt3x +[2020-10-19T17:07:34-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:07:34: Obtaining file (transgene:gencode_peptide_fasta) to the file job store +[2020-10-19T17:07:34-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmplbxp6byq/worker_log.txt +[2020-10-19T17:07:34-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-affo6juu +[2020-10-19T17:07:34-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:07:34: Obtaining file (reports:car_t_targets_file) to the file job store +[2020-10-19T17:07:35-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpkfbxisy1/worker_log.txt +[2020-10-19T17:07:36-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-z1kq216o +[2020-10-19T17:07:36-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:07:36: Obtaining file (transgene:genome_fasta) to the file job store +[2020-10-19T17:07:37-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpzsqvbvg2/worker_log.txt +[2020-10-19T17:07:37-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-sohjmdrx +[2020-10-19T17:07:37-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmprg348wrm/worker_log.txt +[2020-10-19T17:07:38-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:07:38: Obtaining file (indexes:genome_fai) to the file job store +[2020-10-19T17:07:38-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-s7uqh4l5 +[2020-10-19T17:07:38-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpct9dnkyb/worker_log.txt +[2020-10-19T17:07:38-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:07:38: Obtaining file (indexes:genome_dict) to the file job store +[2020-10-19T17:18:26-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-8nav8g_w +[2020-10-19T17:18:27-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:18:27: Obtaining file (radia:dbsnp_beds) to the file job store +[2020-10-19T17:18:27-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp7825aax7/worker_log.txt +[2020-10-19T17:18:49-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6s511qxp +[2020-10-19T17:18:49-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:18:49: Obtaining file (transgene:gencode_transcript_fasta) to the file job store +[2020-10-19T17:18:49-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmph5vjynu5/worker_log.txt +[2020-10-19T17:18:50-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:18:50: Obtaining file (mhcii:method_file) to the file job store +[2020-10-19T17:18:50-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-_tjtgbal +[2020-10-19T17:18:50-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpvwke7f57/worker_log.txt +[2020-10-19T17:19:07-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-88k469b1 +[2020-10-19T17:19:07-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpo3ditjxq/worker_log.txt +[2020-10-19T17:19:07-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:19:07: Obtaining file (radia:cosmic_beds) to the file job store +[2020-10-19T17:19:09-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:19:09: Obtaining file (indexes:dbsnp_idx) to the file job store +[2020-10-19T17:19:09-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-b2p8fmcs +[2020-10-19T17:19:09-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmplljpxctp/worker_log.txt +[2020-10-19T17:28:16-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ue7dbx_y +[2020-10-19T17:28:16-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:28:16: Obtaining file (indexes:dbsnp_vcf) to the file job store +[2020-10-19T17:28:16-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpk54shb5y/worker_log.txt +[2020-10-19T17:28:17-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-2pfe72sp +[2020-10-19T17:28:17-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:28:17: Obtaining file (radia:gencode_beds) to the file job store +[2020-10-19T17:28:17-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp9_00eb1h/worker_log.txt +[2020-10-19T17:28:19-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-7aptjv8m +[2020-10-19T17:28:19-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp7lvzztal/worker_log.txt +[2020-10-19T17:28:19-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:28:19: Obtaining file (indexes:dbsnp_tbi) to the file job store +[2020-10-19T17:33:09-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-zq7r2fof +[2020-10-19T17:33:09-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpydjoimv4/worker_log.txt +[2020-10-19T17:33:09-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:33:09: Obtaining file (bwa:index) to the file job store +[2020-10-19T17:33:10-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1nv69jcb +[2020-10-19T17:33:10-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpxsyao8mc/worker_log.txt +[2020-10-19T17:33:10-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:33:10: Obtaining file (radia:retrogene_beds) to the file job store +[2020-10-19T17:33:11-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1ibo1747 +[2020-10-19T17:33:11-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpukzqga3e/worker_log.txt +[2020-10-19T17:33:11-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:33:11: Obtaining file (radia:pseudogene_beds) to the file job store +[2020-10-19T17:33:11-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-_vdkxdam +[2020-10-19T17:33:11-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpouksc11k/worker_log.txt +[2020-10-19T17:33:11-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:33:11: Obtaining file (reports:mhc_pathways_file) to the file job store +[2020-10-19T17:33:12-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-hbd_web4 +[2020-10-19T17:33:12-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:33:12: Obtaining file (strelka:config_file) to the file job store +[2020-10-19T17:33:12-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmphde5aptl/worker_log.txt +[2020-10-19T17:33:42-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-zf_4y0r9 +[2020-10-19T17:33:42-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp5f11s8cv/worker_log.txt +[2020-10-19T17:33:42-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:33:42: Obtaining file (transgene:gencode_annotation_gtf) to the file job store +[2020-10-19T17:33:43-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-qeqilzdt +[2020-10-19T17:33:43-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:33:43: Obtaining file (mhci:method_file) to the file job store +[2020-10-19T17:38:36-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-vfbtut8x +[2020-10-19T17:38:36-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:38:36: Obtaining file (snpeff:index) to the file job store +[2020-10-19T17:50:10-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-wu4un7pj +[2020-10-19T17:50:10-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 17:50:10: Obtaining file (phlat:index) to the file job store +[2020-10-19T17:55:17-0700] [MainThread] [I] [toil.leader] 1 jobs are running, 0 jobs are issued and waiting to run +[2020-10-19T18:12:27-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-3ot0nqrl +[2020-10-19T18:12:27-0700] [MainThread] [I] [toil.leader] Issued job 'launch_protect' kind-launch_protect/instance-twmlpdal with job batch system ID: 30 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-19T18:12:28-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 18:12:28: Obtaining file (star:index) to the file job store +[2020-10-19T18:12:35-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp_34dbhgg/worker_log.txt +[2020-10-19T18:12:36-0700] [MainThread] [I] [toil.leader] Job ended: 'launch_protect' kind-launch_protect/instance-twmlpdal +[2020-10-19T18:12:36-0700] [MainThread] [I] [toil.leader] Issued job 'prepare_samples' kind-prepare_samples/instance-bgcywx3p with job batch system ID: 31 and cores: 1, disk: 40.0 G, and memory: 2.0 G +[2020-10-19T18:12:36-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpz0f5zfxg/worker_log.txt +[2020-10-19T18:13:27-0700] [MainThread] [I] [toil.leader] Job ended: 'prepare_samples' kind-prepare_samples/instance-bgcywx3p +[2020-10-19T18:13:27-0700] [MainThread] [I] [toil.leader] Issued job 'get_patient_fastqs' kind-get_patient_fastqs/instance-rfimzy94 with job batch system ID: 32 and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-10-19T18:13:27-0700] [MainThread] [I] [toil.leader] Issued job 'get_patient_fastqs' kind-get_patient_fastqs/instance-yd9qq2y6 with job batch system ID: 33 and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-10-19T18:13:27-0700] [MainThread] [I] [toil.leader] Issued job 'get_patient_fastqs' kind-get_patient_fastqs/instance-gz400tcp with job batch system ID: 34 and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-10-19T18:13:27-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 18:13:27: Downloading Inputs for TEST +[2020-10-19T18:13:27-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 18:13:27: Obtaining file (TEST:tumor_dna_fastq_2) to the file job store +[2020-10-19T18:13:27-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 18:13:27: Obtaining file (TEST:normal_dna_fastq_1) to the file job store +[2020-10-19T18:13:27-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 18:13:27: Obtaining file (TEST:tumor_dna_fastq_1) to the file job store +[2020-10-19T18:13:27-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 18:13:27: Obtaining file (TEST:tumor_rna_fastq_1) to the file job store +[2020-10-19T18:13:27-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 18:13:27: Obtaining file (TEST:normal_dna_fastq_2) to the file job store +[2020-10-19T18:13:27-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 18:13:27: Obtaining file (TEST:tumor_rna_fastq_2) to the file job store +[2020-10-19T18:13:27-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp0h42moby/worker_log.txt +[2020-10-19T18:13:27-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpwlju589m/worker_log.txt +[2020-10-19T18:13:27-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpcy8tnpe6/worker_log.txt +[2020-10-19T18:13:27-0700] [MainThread] [I] [toil.leader] Job ended: 'get_patient_fastqs' kind-get_patient_fastqs/instance-yd9qq2y6 +[2020-10-19T18:13:27-0700] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-71c26qug with job batch system ID: 35 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-10-19T18:13:27-0700] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-jm64shh5 with job batch system ID: 36 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.leader] Job ended: 'get_patient_fastqs' kind-get_patient_fastqs/instance-rfimzy94 +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-cmnhiln5 with job batch system ID: 37 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-0yywb779 with job batch system ID: 38 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.leader] Job ended: 'get_patient_fastqs' kind-get_patient_fastqs/instance-gz400tcp +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-gbwokb12 with job batch system ID: 39 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-zbsn_cy4 with job batch system ID: 40 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpvdbx8kkz/worker_log.txt +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-71c26qug +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-j5t5f3ax with job batch system ID: 41 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpg7fkyjah/worker_log.txt +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpckug2acy/worker_log.txt +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpbger8qyf/worker_log.txt +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpgl094r20/worker_log.txt +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp8r1vs78f/worker_log.txt +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-jm64shh5 +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-nl7qvmik with job batch system ID: 42 and cores: 4, disk: 9.4 G, and memory: 12.0 G +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-cmnhiln5 +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-wyy831pe with job batch system ID: 43 and cores: 4, disk: 7.8 G, and memory: 2.0 G +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-gbwokb12 +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-dyn6bo3z with job batch system ID: 44 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-0yywb779 +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-39qi7a25 with job batch system ID: 45 and cores: 4, disk: 9.6 G, and memory: 12.0 G +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-zbsn_cy4 +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.leader] Issued job 'run_cutadapt' kind-run_cutadapt/instance-aop8fhhj with job batch system ID: 46 and cores: 1, disk: 539.2 M, and memory: 2.0 G +[2020-10-19T18:13:28-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpd8hvoy4h/worker_log.txt +[2020-10-19T18:14:57-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_bwa file:/home/drkthomp/protect3/jobStore kind-run_bwa/instance-nl7qvmik. +[2020-10-19T18:14:57-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_bwa' kind-run_bwa/instance-nl7qvmik +[2020-10-19T18:14:57-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_bwa' kind-run_bwa/instance-nl7qvmik +[2020-10-19T18:14:57-0700] [MainThread] [W] [toil.leader] Log from job kind-run_bwa/instance-nl7qvmik follows: +=========> + [2020-10-19T18:13:28-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T18:13:28-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/alignment/dna.py", line 140, in run_bwa + docker_call(tool='bwa', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 101, in docker_call + assert isinstance(outfile, file), 'outfile was not passsed a file' + NameError: name 'file' is not defined + [2020-10-19T18:14:56-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-19T18:14:57-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_bwa' kind-run_bwa/instance-nl7qvmik with ID kind-run_bwa/instance-nl7qvmik to 1 +[2020-10-19T18:14:57-0700] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-nl7qvmik with job batch system ID: 47 and cores: 4, disk: 9.4 G, and memory: 12.0 G +[2020-10-19T18:14:58-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmphqh2kspz/worker_log.txt +[2020-10-19T18:17:57-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/home/drkthomp/protect3/jobStore kind-run_phlat/instance-dyn6bo3z. +[2020-10-19T18:17:57-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-dyn6bo3z +[2020-10-19T18:17:57-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-dyn6bo3z +[2020-10-19T18:17:57-0700] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-dyn6bo3z follows: +=========> + [2020-10-19T18:14:58-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T18:14:58-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + 462668 reads; of these: + 462668 (100.00%) were paired; of these: + 75207 (16.26%) aligned concordantly 0 times + 235430 (50.89%) aligned concordantly exactly 1 time + 152031 (32.86%) aligned concordantly >1 times + 83.74% overall alignment rate + .....Process Bowtie 2 mapping on tumor_rna....... + + .....Prepare files of tumor_rna for PHLAT....... + + ..... Running PHLAT ....... + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 64, in + go(tag,outdir,phlatdir) + File "/home/yubai/programs/phlat-1.0/dist/utilities.py", line 79, in go + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 617, in __init__ + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 645, in setMaster + IndexError: list index out of range + Traceback (most recent call last): + File "/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib/python3.8/subprocess.py", line 364, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmphqh2kspz/d29e0646-f21a-49aa-a135-1795672538fc:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_rna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '4']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 73, in run_phlat + docker_call(tool='phlat', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 138, in docker_call + raise RuntimeError('docker command returned a non-zero exit status (%s)' % err.returncode + + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmphqh2kspz/d29e0646-f21a-49aa-a135-1795672538fc:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_rna -e /home/phlat-1.0 -o /data -p 4" + [2020-10-19T18:17:57-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox + {'input_1.fastq': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmphqh2kspz/d29e0646-f21a-49aa-a135-1795672538fc/input_1.fastq', 'input_2.fastq': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmphqh2kspz/d29e0646-f21a-49aa-a135-1795672538fc/input_2.fastq', 'phlat_index.tar.gz': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmphqh2kspz/d29e0646-f21a-49aa-a135-1795672538fc/phlat_index.tar.gz'} +<========= +[2020-10-19T18:17:57-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-dyn6bo3z with ID kind-run_phlat/instance-dyn6bo3z to 1 +[2020-10-19T18:17:57-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-dyn6bo3z with job batch system ID: 48 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-19T18:17:59-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp9ghrqz2a/worker_log.txt +[2020-10-19T18:19:17-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_bwa file:/home/drkthomp/protect3/jobStore kind-run_bwa/instance-39qi7a25. +[2020-10-19T18:19:17-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_bwa' kind-run_bwa/instance-39qi7a25 +[2020-10-19T18:19:17-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_bwa' kind-run_bwa/instance-39qi7a25 +[2020-10-19T18:19:17-0700] [MainThread] [W] [toil.leader] Log from job kind-run_bwa/instance-39qi7a25 follows: +=========> + [2020-10-19T18:17:59-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T18:17:59-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/alignment/dna.py", line 140, in run_bwa + docker_call(tool='bwa', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 101, in docker_call + assert isinstance(outfile, file), 'outfile was not passsed a file' + NameError: name 'file' is not defined + [2020-10-19T18:19:16-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-19T18:19:17-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_bwa' kind-run_bwa/instance-39qi7a25 with ID kind-run_bwa/instance-39qi7a25 to 1 +[2020-10-19T18:19:17-0700] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-39qi7a25 with job batch system ID: 49 and cores: 4, disk: 9.6 G, and memory: 12.0 G +[2020-10-19T18:19:18-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpr7_rtzjc/worker_log.txt +[2020-10-19T18:20:13-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_bwa file:/home/drkthomp/protect3/jobStore kind-run_bwa/instance-nl7qvmik. +[2020-10-19T18:20:13-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_bwa' kind-run_bwa/instance-nl7qvmik +[2020-10-19T18:20:13-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_bwa' kind-run_bwa/instance-nl7qvmik +[2020-10-19T18:20:13-0700] [MainThread] [W] [toil.leader] Log from job kind-run_bwa/instance-nl7qvmik follows: +=========> + [2020-10-19T18:19:18-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T18:19:18-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/alignment/dna.py", line 140, in run_bwa + docker_call(tool='bwa', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 101, in docker_call + assert isinstance(outfile, file), 'outfile was not passsed a file' + NameError: name 'file' is not defined + [2020-10-19T18:20:12-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-19T18:20:13-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_bwa' kind-run_bwa/instance-nl7qvmik with ID kind-run_bwa/instance-nl7qvmik to 0 +[2020-10-19T18:20:13-0700] [MainThread] [W] [toil.leader] Job 'run_bwa' kind-run_bwa/instance-nl7qvmik with ID kind-run_bwa/instance-nl7qvmik is completely failed +[2020-10-19T18:20:13-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp8j4mqmrx/worker_log.txt +[2020-10-19T18:22:49-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/home/drkthomp/protect3/jobStore kind-run_phlat/instance-dyn6bo3z. +[2020-10-19T18:22:49-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-dyn6bo3z +[2020-10-19T18:22:49-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-dyn6bo3z +[2020-10-19T18:22:49-0700] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-dyn6bo3z follows: +=========> + [2020-10-19T18:20:13-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T18:20:13-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + 462668 reads; of these: + 462668 (100.00%) were paired; of these: + 75207 (16.26%) aligned concordantly 0 times + 235430 (50.89%) aligned concordantly exactly 1 time + 152031 (32.86%) aligned concordantly >1 times + 83.74% overall alignment rate + .....Process Bowtie 2 mapping on tumor_rna....... + + .....Prepare files of tumor_rna for PHLAT....... + + ..... Running PHLAT ....... + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 64, in + go(tag,outdir,phlatdir) + File "/home/yubai/programs/phlat-1.0/dist/utilities.py", line 79, in go + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 617, in __init__ + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 645, in setMaster + IndexError: list index out of range + Traceback (most recent call last): + File "/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib/python3.8/subprocess.py", line 364, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp8j4mqmrx/d7cf399c-7991-44fc-9763-b75671fb50d4:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_rna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '4']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 73, in run_phlat + docker_call(tool='phlat', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 138, in docker_call + raise RuntimeError('docker command returned a non-zero exit status (%s)' % err.returncode + + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp8j4mqmrx/d7cf399c-7991-44fc-9763-b75671fb50d4:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_rna -e /home/phlat-1.0 -o /data -p 4" + [2020-10-19T18:22:48-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox + {'input_1.fastq': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp8j4mqmrx/d7cf399c-7991-44fc-9763-b75671fb50d4/input_1.fastq', 'input_2.fastq': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp8j4mqmrx/d7cf399c-7991-44fc-9763-b75671fb50d4/input_2.fastq', 'phlat_index.tar.gz': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp8j4mqmrx/d7cf399c-7991-44fc-9763-b75671fb50d4/phlat_index.tar.gz'} +<========= +[2020-10-19T18:22:49-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-dyn6bo3z with ID kind-run_phlat/instance-dyn6bo3z to 0 +[2020-10-19T18:22:49-0700] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-dyn6bo3z with ID kind-run_phlat/instance-dyn6bo3z is completely failed +[2020-10-19T18:22:49-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpzj2ufyd8/worker_log.txt +[2020-10-19T18:26:16-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/home/drkthomp/protect3/jobStore kind-run_phlat/instance-wyy831pe. +[2020-10-19T18:26:16-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-wyy831pe +[2020-10-19T18:26:16-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-wyy831pe +[2020-10-19T18:26:16-0700] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-wyy831pe follows: +=========> + [2020-10-19T18:22:49-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T18:22:49-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + 567573 reads; of these: + 567573 (100.00%) were paired; of these: + 3570 (0.63%) aligned concordantly 0 times + 514680 (90.68%) aligned concordantly exactly 1 time + 49323 (8.69%) aligned concordantly >1 times + 99.37% overall alignment rate + .....Process Bowtie 2 mapping on tumor_dna....... + + .....Prepare files of tumor_dna for PHLAT....... + + ..... Running PHLAT ....... + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 64, in + go(tag,outdir,phlatdir) + File "/home/yubai/programs/phlat-1.0/dist/utilities.py", line 79, in go + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 617, in __init__ + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 645, in setMaster + IndexError: list index out of range + Traceback (most recent call last): + File "/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib/python3.8/subprocess.py", line 364, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpzj2ufyd8/11b68275-efb9-4da5-b092-8fc06a28bd71:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '4']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 73, in run_phlat + docker_call(tool='phlat', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 138, in docker_call + raise RuntimeError('docker command returned a non-zero exit status (%s)' % err.returncode + + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpzj2ufyd8/11b68275-efb9-4da5-b092-8fc06a28bd71:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_dna -e /home/phlat-1.0 -o /data -p 4" + [2020-10-19T18:26:16-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox + {'input_1.fastq': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpzj2ufyd8/11b68275-efb9-4da5-b092-8fc06a28bd71/input_1.fastq', 'input_2.fastq': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpzj2ufyd8/11b68275-efb9-4da5-b092-8fc06a28bd71/input_2.fastq', 'phlat_index.tar.gz': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpzj2ufyd8/11b68275-efb9-4da5-b092-8fc06a28bd71/phlat_index.tar.gz'} +<========= +[2020-10-19T18:26:16-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-wyy831pe with ID kind-run_phlat/instance-wyy831pe to 1 +[2020-10-19T18:26:16-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-wyy831pe with job batch system ID: 50 and cores: 4, disk: 7.8 G, and memory: 2.0 G +[2020-10-19T18:26:16-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpd_baalno/worker_log.txt +[2020-10-19T18:29:19-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/home/drkthomp/protect3/jobStore kind-run_phlat/instance-j5t5f3ax. +[2020-10-19T18:29:19-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-j5t5f3ax +[2020-10-19T18:29:19-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-j5t5f3ax +[2020-10-19T18:29:19-0700] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-j5t5f3ax follows: +=========> + [2020-10-19T18:26:16-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T18:26:16-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + 491247 reads; of these: + 491247 (100.00%) were paired; of these: + 3488 (0.71%) aligned concordantly 0 times + 442898 (90.16%) aligned concordantly exactly 1 time + 44861 (9.13%) aligned concordantly >1 times + 99.29% overall alignment rate + .....Process Bowtie 2 mapping on normal_dna....... + + .....Prepare files of normal_dna for PHLAT....... + + ..... Running PHLAT ....... + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 64, in + go(tag,outdir,phlatdir) + File "/home/yubai/programs/phlat-1.0/dist/utilities.py", line 79, in go + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 617, in __init__ + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 645, in setMaster + IndexError: list index out of range + Traceback (most recent call last): + File "/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib/python3.8/subprocess.py", line 364, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpd_baalno/ba1f4175-9246-4334-ba0a-fe8d2f8e7b38:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'normal_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '4']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 73, in run_phlat + docker_call(tool='phlat', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 138, in docker_call + raise RuntimeError('docker command returned a non-zero exit status (%s)' % err.returncode + + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpd_baalno/ba1f4175-9246-4334-ba0a-fe8d2f8e7b38:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag normal_dna -e /home/phlat-1.0 -o /data -p 4" + [2020-10-19T18:29:19-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox + {'input_1.fastq': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpd_baalno/ba1f4175-9246-4334-ba0a-fe8d2f8e7b38/input_1.fastq', 'input_2.fastq': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpd_baalno/ba1f4175-9246-4334-ba0a-fe8d2f8e7b38/input_2.fastq', 'phlat_index.tar.gz': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpd_baalno/ba1f4175-9246-4334-ba0a-fe8d2f8e7b38/phlat_index.tar.gz'} +<========= +[2020-10-19T18:29:19-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-j5t5f3ax with ID kind-run_phlat/instance-j5t5f3ax to 1 +[2020-10-19T18:29:19-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-j5t5f3ax with job batch system ID: 51 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-19T18:29:20-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpevrcd_19/worker_log.txt +[2020-10-19T18:32:41-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/home/drkthomp/protect3/jobStore kind-run_phlat/instance-wyy831pe. +[2020-10-19T18:32:41-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-wyy831pe +[2020-10-19T18:32:41-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-wyy831pe +[2020-10-19T18:32:41-0700] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-wyy831pe follows: +=========> + [2020-10-19T18:29:20-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T18:29:20-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + 567573 reads; of these: + 567573 (100.00%) were paired; of these: + 3570 (0.63%) aligned concordantly 0 times + 514680 (90.68%) aligned concordantly exactly 1 time + 49323 (8.69%) aligned concordantly >1 times + 99.37% overall alignment rate + .....Process Bowtie 2 mapping on tumor_dna....... + + .....Prepare files of tumor_dna for PHLAT....... + + ..... Running PHLAT ....... + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 64, in + go(tag,outdir,phlatdir) + File "/home/yubai/programs/phlat-1.0/dist/utilities.py", line 79, in go + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 617, in __init__ + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 645, in setMaster + IndexError: list index out of range + Traceback (most recent call last): + File "/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib/python3.8/subprocess.py", line 364, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpevrcd_19/aaf341a0-836d-4fdb-ad14-ecc46c8582ed:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '4']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 73, in run_phlat + docker_call(tool='phlat', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 138, in docker_call + raise RuntimeError('docker command returned a non-zero exit status (%s)' % err.returncode + + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpevrcd_19/aaf341a0-836d-4fdb-ad14-ecc46c8582ed:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_dna -e /home/phlat-1.0 -o /data -p 4" + [2020-10-19T18:32:41-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox + {'input_1.fastq': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpevrcd_19/aaf341a0-836d-4fdb-ad14-ecc46c8582ed/input_1.fastq', 'input_2.fastq': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpevrcd_19/aaf341a0-836d-4fdb-ad14-ecc46c8582ed/input_2.fastq', 'phlat_index.tar.gz': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpevrcd_19/aaf341a0-836d-4fdb-ad14-ecc46c8582ed/phlat_index.tar.gz'} +<========= +[2020-10-19T18:32:41-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-wyy831pe with ID kind-run_phlat/instance-wyy831pe to 0 +[2020-10-19T18:32:41-0700] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-wyy831pe with ID kind-run_phlat/instance-wyy831pe is completely failed +[2020-10-19T18:32:41-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpjevzqsps/worker_log.txt +[2020-10-19T18:33:38-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_bwa file:/home/drkthomp/protect3/jobStore kind-run_bwa/instance-39qi7a25. +[2020-10-19T18:33:38-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_bwa' kind-run_bwa/instance-39qi7a25 +[2020-10-19T18:33:38-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_bwa' kind-run_bwa/instance-39qi7a25 +[2020-10-19T18:33:38-0700] [MainThread] [W] [toil.leader] Log from job kind-run_bwa/instance-39qi7a25 follows: +=========> + [2020-10-19T18:32:41-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T18:32:41-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/alignment/dna.py", line 140, in run_bwa + docker_call(tool='bwa', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 101, in docker_call + assert isinstance(outfile, file), 'outfile was not passsed a file' + NameError: name 'file' is not defined + [2020-10-19T18:33:37-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-19T18:33:38-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_bwa' kind-run_bwa/instance-39qi7a25 with ID kind-run_bwa/instance-39qi7a25 to 0 +[2020-10-19T18:33:38-0700] [MainThread] [W] [toil.leader] Job 'run_bwa' kind-run_bwa/instance-39qi7a25 with ID kind-run_bwa/instance-39qi7a25 is completely failed +[2020-10-19T18:33:39-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpbit8qnxo/worker_log.txt +[2020-10-19T18:33:50-0700] [MainThread] [I] [toil.leader] Job ended: 'run_cutadapt' kind-run_cutadapt/instance-aop8fhhj +[2020-10-19T18:33:50-0700] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-pj1938zl with job batch system ID: 52 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-10-19T18:33:51-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpk71upqs5/worker_log.txt +[2020-10-19T18:33:51-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-19-2020 18:33:51: Ran cutadapt on TEST successfully +[2020-10-19T18:37:24-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/home/drkthomp/protect3/jobStore kind-run_phlat/instance-j5t5f3ax. +[2020-10-19T18:37:24-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-j5t5f3ax +[2020-10-19T18:37:24-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-j5t5f3ax +[2020-10-19T18:37:24-0700] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-j5t5f3ax follows: +=========> + [2020-10-19T18:33:51-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-19T18:33:51-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + 491247 reads; of these: + 491247 (100.00%) were paired; of these: + 3488 (0.71%) aligned concordantly 0 times + 442898 (90.16%) aligned concordantly exactly 1 time + 44861 (9.13%) aligned concordantly >1 times + 99.29% overall alignment rate + .....Process Bowtie 2 mapping on normal_dna....... + + .....Prepare files of normal_dna for PHLAT....... + + ..... Running PHLAT ....... + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 64, in + go(tag,outdir,phlatdir) + File "/home/yubai/programs/phlat-1.0/dist/utilities.py", line 79, in go + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 617, in __init__ + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 645, in setMaster + IndexError: list index out of range + Traceback (most recent call last): + File "/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib/python3.8/subprocess.py", line 364, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpk71upqs5/a37cc46e-5f9c-48f0-9c61-db0a80864e05:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'normal_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '4']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 73, in run_phlat + docker_call(tool='phlat', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 138, in docker_call + raise RuntimeError('docker command returned a non-zero exit status (%s)' % err.returncode + + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpk71upqs5/a37cc46e-5f9c-48f0-9c61-db0a80864e05:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag normal_dna -e /home/phlat-1.0 -o /data -p 4" + [2020-10-19T18:37:24-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox + {'input_1.fastq': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpk71upqs5/a37cc46e-5f9c-48f0-9c61-db0a80864e05/input_1.fastq', 'input_2.fastq': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpk71upqs5/a37cc46e-5f9c-48f0-9c61-db0a80864e05/input_2.fastq', 'phlat_index.tar.gz': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpk71upqs5/a37cc46e-5f9c-48f0-9c61-db0a80864e05/phlat_index.tar.gz'} +<========= +[2020-10-19T18:37:24-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-j5t5f3ax with ID kind-run_phlat/instance-j5t5f3ax to 0 +[2020-10-19T18:37:24-0700] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-j5t5f3ax with ID kind-run_phlat/instance-j5t5f3ax is completely failed +[2020-10-19T18:37:25-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpybqhu9re/worker_log.txt +[2020-10-19T18:37:25-0700] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-pj1938zl +[2020-10-19T18:37:25-0700] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-bwrjybkc with job batch system ID: 53 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-10-19T18:37:25-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmprpemv6o8/worker_log.txt +[2020-10-19T18:37:26-0700] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-bwrjybkc +[2020-10-19T18:37:26-0700] [MainThread] [I] [toil.leader] Issued job 'run_star' kind-run_star/instance-5q77j6ix with job batch system ID: 54 and cores: 4, disk: 52.8 G, and memory: 48.6 G +[2020-10-19T18:55:18-0700] [MainThread] [I] [toil.leader] 0 jobs are running, 1 jobs are issued and waiting to run +[2020-10-19T19:55:19-0700] [MainThread] [I] [toil.leader] 0 jobs are running, 1 jobs are issued and waiting to run +[2020-10-19T20:55:20-0700] [MainThread] [I] [toil.leader] 0 jobs are running, 1 jobs are issued and waiting to run +[2020-10-19T21:55:22-0700] [MainThread] [I] [toil.leader] 0 jobs are running, 1 jobs are issued and waiting to run diff --git a/errors/2020-10-20-19-53-59.txt b/errors/2020-10-20-19-53-59.txt new file mode 100644 index 00000000..0190f9c1 --- /dev/null +++ b/errors/2020-10-20-19-53-59.txt @@ -0,0 +1,65 @@ +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-p2km7zbq +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-8gyqgqyx +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ourgk6vm +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-bm4dpfik +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-0p9602y6 +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-70qhj097 +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-yrvj0gxx +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-tjkov9ea +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-gl_w17lv +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-cev_msom +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-0t3dxg78 +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-8neyzru4 +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-c65720sq +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-cije5tr0 +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-daisoolw +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-9hdk8gj3 +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-92gnpm9m +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-vodde7jc +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-wyy831pe +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-jm64shh5 +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-6clikwgf +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-h18ep_9e +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-40bfs1db +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-j5t5f3ax +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-cp3zlhzq +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-3a6rn9f9 +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-dyn6bo3z +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-0yywb779 +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-55hd3p6y +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-fu63ea_u +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-0oyi7ie_ +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-wflniclq +[2020-10-20T19:54:01-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-sfhpnee3 +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-wyy831pe with job batch system ID: 0 and cores: 4, disk: 7.8 G, and memory: 2.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-j5t5f3ax with job batch system ID: 1 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-8gyqgqyx with job batch system ID: 2 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-55hd3p6y with job batch system ID: 3 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cije5tr0 with job batch system ID: 4 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cp3zlhzq with job batch system ID: 5 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h18ep_9e with job batch system ID: 6 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-vodde7jc with job batch system ID: 7 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-40bfs1db with job batch system ID: 8 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-92gnpm9m with job batch system ID: 9 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gl_w17lv with job batch system ID: 10 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-6clikwgf with job batch system ID: 11 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0p9602y6 with job batch system ID: 12 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-tjkov9ea with job batch system ID: 13 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9hdk8gj3 with job batch system ID: 14 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cev_msom with job batch system ID: 15 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fu63ea_u with job batch system ID: 16 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0oyi7ie_ with job batch system ID: 17 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ourgk6vm with job batch system ID: 18 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-3a6rn9f9 with job batch system ID: 19 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-sfhpnee3 with job batch system ID: 20 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bm4dpfik with job batch system ID: 21 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0t3dxg78 with job batch system ID: 22 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-daisoolw with job batch system ID: 23 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-c65720sq with job batch system ID: 24 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-wflniclq with job batch system ID: 25 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yrvj0gxx with job batch system ID: 26 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-dyn6bo3z with job batch system ID: 27 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_star' kind-run_star/instance-5q77j6ix with job batch system ID: 28 and cores: 4, disk: 52.8 G, and memory: 48.6 G +[2020-10-20T19:54:01-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpne_33r5r/worker_log.txt +[2020-10-20T19:54:03-0700] [MainThread] [I] [toil.leader] 1 jobs are running, 28 jobs are issued and waiting to run diff --git a/errors/2020-10-20-19-55-00.txt b/errors/2020-10-20-19-55-00.txt new file mode 100644 index 00000000..dc21ad93 --- /dev/null +++ b/errors/2020-10-20-19-55-00.txt @@ -0,0 +1,1387 @@ +[2020-10-20T19:55:03-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-wyy831pe +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-wyy831pe with job batch system ID: 0 and cores: 4, disk: 7.8 G, and memory: 2.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-j5t5f3ax with job batch system ID: 1 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-8gyqgqyx with job batch system ID: 2 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-55hd3p6y with job batch system ID: 3 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cije5tr0 with job batch system ID: 4 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cp3zlhzq with job batch system ID: 5 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h18ep_9e with job batch system ID: 6 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-vodde7jc with job batch system ID: 7 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-40bfs1db with job batch system ID: 8 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-92gnpm9m with job batch system ID: 9 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gl_w17lv with job batch system ID: 10 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-6clikwgf with job batch system ID: 11 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0p9602y6 with job batch system ID: 12 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-tjkov9ea with job batch system ID: 13 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9hdk8gj3 with job batch system ID: 14 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cev_msom with job batch system ID: 15 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fu63ea_u with job batch system ID: 16 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0oyi7ie_ with job batch system ID: 17 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ourgk6vm with job batch system ID: 18 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-3a6rn9f9 with job batch system ID: 19 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-sfhpnee3 with job batch system ID: 20 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bm4dpfik with job batch system ID: 21 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0t3dxg78 with job batch system ID: 22 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-daisoolw with job batch system ID: 23 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-c65720sq with job batch system ID: 24 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-wflniclq with job batch system ID: 25 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yrvj0gxx with job batch system ID: 26 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-dyn6bo3z with job batch system ID: 27 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-20T19:55:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_star' kind-run_star/instance-5q77j6ix with job batch system ID: 28 and cores: 4, disk: 52.8 G, and memory: 48.6 G +[2020-10-20T19:55:04-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpzq2p34mq/worker_log.txt +[2020-10-20T19:55:05-0700] [MainThread] [I] [toil.leader] 1 jobs are running, 28 jobs are issued and waiting to run +[2020-10-20T19:59:53-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/home/drkthomp/protect3/jobStore kind-run_phlat/instance-wyy831pe. +[2020-10-20T19:59:53-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-wyy831pe +[2020-10-20T19:59:53-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-wyy831pe +[2020-10-20T19:59:53-0700] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-wyy831pe follows: +=========> + [2020-10-20T19:55:04-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:55:04-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + 567573 reads; of these: + 567573 (100.00%) were paired; of these: + 3570 (0.63%) aligned concordantly 0 times + 514680 (90.68%) aligned concordantly exactly 1 time + 49323 (8.69%) aligned concordantly >1 times + 99.37% overall alignment rate + .....Process Bowtie 2 mapping on tumor_dna....... + + .....Prepare files of tumor_dna for PHLAT....... + + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 64, in + ..... Running PHLAT ....... + go(tag,outdir,phlatdir) + File "/home/yubai/programs/phlat-1.0/dist/utilities.py", line 79, in go + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 617, in __init__ + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 645, in setMaster + IndexError: list index out of range + Traceback (most recent call last): + File "/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib/python3.8/subprocess.py", line 364, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpzq2p34mq/dc8ad8e1-8a53-4203-a6be-af90156e5ece:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '4']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 73, in run_phlat + docker_call(tool='phlat', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 138, in docker_call + raise RuntimeError('docker command returned a non-zero exit status (%s)' % err.returncode + + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpzq2p34mq/dc8ad8e1-8a53-4203-a6be-af90156e5ece:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_dna -e /home/phlat-1.0 -o /data -p 4" + [2020-10-20T19:59:53-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox + {'input_1.fastq': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpzq2p34mq/dc8ad8e1-8a53-4203-a6be-af90156e5ece/input_1.fastq', 'input_2.fastq': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpzq2p34mq/dc8ad8e1-8a53-4203-a6be-af90156e5ece/input_2.fastq', 'phlat_index.tar.gz': '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpzq2p34mq/dc8ad8e1-8a53-4203-a6be-af90156e5ece/phlat_index.tar.gz'} +<========= +[2020-10-20T19:59:53-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-wyy831pe with ID kind-run_phlat/instance-wyy831pe to 1 +[2020-10-20T19:59:53-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-wyy831pe with job batch system ID: 29 and cores: 4, disk: 7.8 G, and memory: 2.0 G +[2020-10-20T19:59:55-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmph6pezca5/worker_log.txt +[2020-10-20T19:59:55-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpo_180tf9/worker_log.txt +[2020-10-20T19:59:55-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-fu63ea_u. +[2020-10-20T19:59:55-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fu63ea_u +[2020-10-20T19:59:55-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fu63ea_u +[2020-10-20T19:59:55-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-fu63ea_u follows: +=========> + [2020-10-20T19:59:55-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:55-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:55-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:55-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fu63ea_u with ID kind-run_mutect_perchrom/instance-fu63ea_u to 1 +[2020-10-20T19:59:55-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fu63ea_u with job batch system ID: 30 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:59:55-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-0oyi7ie_. +[2020-10-20T19:59:55-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0oyi7ie_ +[2020-10-20T19:59:55-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0oyi7ie_ +[2020-10-20T19:59:55-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-0oyi7ie_ follows: +=========> + [2020-10-20T19:59:55-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:55-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:55-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:55-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0oyi7ie_ with ID kind-run_mutect_perchrom/instance-0oyi7ie_ to 1 +[2020-10-20T19:59:55-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0oyi7ie_ with job batch system ID: 31 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:59:56-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpp3s46i2h/worker_log.txt +[2020-10-20T19:59:56-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpl_mg14pv/worker_log.txt +[2020-10-20T19:59:56-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-sfhpnee3. +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-sfhpnee3 +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-sfhpnee3 +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-sfhpnee3 follows: +=========> + [2020-10-20T19:59:56-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:56-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:56-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-sfhpnee3 with ID kind-run_mutect_perchrom/instance-sfhpnee3 to 1 +[2020-10-20T19:59:56-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-sfhpnee3 with job batch system ID: 32 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:59:56-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-daisoolw. +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-daisoolw +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-daisoolw +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-daisoolw follows: +=========> + [2020-10-20T19:59:56-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:56-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:56-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-daisoolw with ID kind-run_mutect_perchrom/instance-daisoolw to 1 +[2020-10-20T19:59:56-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-daisoolw with job batch system ID: 33 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:59:56-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpzczpuoh0/worker_log.txt +[2020-10-20T19:59:56-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpzcuw8cy_/worker_log.txt +[2020-10-20T19:59:56-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-8gyqgqyx. +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-8gyqgqyx +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-8gyqgqyx +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-8gyqgqyx follows: +=========> + [2020-10-20T19:59:56-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:56-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:56-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-8gyqgqyx with ID kind-run_mutect_perchrom/instance-8gyqgqyx to 1 +[2020-10-20T19:59:56-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-8gyqgqyx with job batch system ID: 34 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:59:56-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-cp3zlhzq. +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cp3zlhzq +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cp3zlhzq +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-cp3zlhzq follows: +=========> + [2020-10-20T19:59:56-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:56-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:56-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cp3zlhzq with ID kind-run_mutect_perchrom/instance-cp3zlhzq to 1 +[2020-10-20T19:59:56-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cp3zlhzq with job batch system ID: 35 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:59:56-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpug3kd1d1/worker_log.txt +[2020-10-20T19:59:56-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpuhdsryhg/worker_log.txt +[2020-10-20T19:59:56-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-0p9602y6. +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0p9602y6 +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0p9602y6 +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-0p9602y6 follows: +=========> + [2020-10-20T19:59:56-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:56-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:56-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0p9602y6 with ID kind-run_mutect_perchrom/instance-0p9602y6 to 1 +[2020-10-20T19:59:56-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0p9602y6 with job batch system ID: 36 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:59:56-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-9hdk8gj3. +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9hdk8gj3 +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9hdk8gj3 +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-9hdk8gj3 follows: +=========> + [2020-10-20T19:59:56-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:56-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:56-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:56-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9hdk8gj3 with ID kind-run_mutect_perchrom/instance-9hdk8gj3 to 1 +[2020-10-20T19:59:56-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9hdk8gj3 with job batch system ID: 37 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:59:57-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpw5juq_26/worker_log.txt +[2020-10-20T19:59:57-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpqq4z20t9/worker_log.txt +[2020-10-20T19:59:57-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-ourgk6vm. +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ourgk6vm +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ourgk6vm +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-ourgk6vm follows: +=========> + [2020-10-20T19:59:57-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:57-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:57-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ourgk6vm with ID kind-run_mutect_perchrom/instance-ourgk6vm to 1 +[2020-10-20T19:59:57-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ourgk6vm with job batch system ID: 38 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:59:57-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-0t3dxg78. +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0t3dxg78 +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0t3dxg78 +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-0t3dxg78 follows: +=========> + [2020-10-20T19:59:57-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:57-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:57-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0t3dxg78 with ID kind-run_mutect_perchrom/instance-0t3dxg78 to 1 +[2020-10-20T19:59:57-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0t3dxg78 with job batch system ID: 39 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:59:57-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpjgnmciut/worker_log.txt +[2020-10-20T19:59:57-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-c65720sq. +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-c65720sq +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-c65720sq +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-c65720sq follows: +=========> + [2020-10-20T19:59:57-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:57-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:57-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-c65720sq with ID kind-run_mutect_perchrom/instance-c65720sq to 1 +[2020-10-20T19:59:57-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-c65720sq with job batch system ID: 40 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:59:57-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpri6pgo4b/worker_log.txt +[2020-10-20T19:59:57-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-55hd3p6y. +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-55hd3p6y +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-55hd3p6y +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-55hd3p6y follows: +=========> + [2020-10-20T19:59:57-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:57-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:57-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-55hd3p6y with ID kind-run_mutect_perchrom/instance-55hd3p6y to 1 +[2020-10-20T19:59:57-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-55hd3p6y with job batch system ID: 41 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:59:57-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp_akv79mp/worker_log.txt +[2020-10-20T19:59:57-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpoctwff98/worker_log.txt +[2020-10-20T19:59:57-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-sfhpnee3. +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-sfhpnee3 +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-sfhpnee3 +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-sfhpnee3 follows: +=========> + [2020-10-20T19:59:57-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:57-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:57-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-sfhpnee3 with ID kind-run_mutect_perchrom/instance-sfhpnee3 to 0 +[2020-10-20T19:59:57-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-sfhpnee3 with ID kind-run_mutect_perchrom/instance-sfhpnee3 is completely failed +[2020-10-20T19:59:58-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-vodde7jc. +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-vodde7jc +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-vodde7jc +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-vodde7jc follows: +=========> + [2020-10-20T19:59:57-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:57-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:57-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-vodde7jc with ID kind-run_mutect_perchrom/instance-vodde7jc to 1 +[2020-10-20T19:59:58-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-vodde7jc with job batch system ID: 42 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:59:58-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp2haaw2c4/worker_log.txt +[2020-10-20T19:59:58-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmph2rur90m/worker_log.txt +[2020-10-20T19:59:58-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-55hd3p6y. +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-55hd3p6y +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-55hd3p6y +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-55hd3p6y follows: +=========> + [2020-10-20T19:59:58-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:58-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:58-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-55hd3p6y with ID kind-run_mutect_perchrom/instance-55hd3p6y to 0 +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-55hd3p6y with ID kind-run_mutect_perchrom/instance-55hd3p6y is completely failed +[2020-10-20T19:59:58-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-gl_w17lv. +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gl_w17lv +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gl_w17lv +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-gl_w17lv follows: +=========> + [2020-10-20T19:59:58-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:58-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:58-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gl_w17lv with ID kind-run_mutect_perchrom/instance-gl_w17lv to 1 +[2020-10-20T19:59:58-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gl_w17lv with job batch system ID: 43 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:59:58-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpqcjs5siu/worker_log.txt +[2020-10-20T19:59:58-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpolf2dxb6/worker_log.txt +[2020-10-20T19:59:58-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-92gnpm9m. +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-92gnpm9m +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-92gnpm9m +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-92gnpm9m follows: +=========> + [2020-10-20T19:59:58-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:58-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:58-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-92gnpm9m with ID kind-run_mutect_perchrom/instance-92gnpm9m to 1 +[2020-10-20T19:59:58-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-92gnpm9m with job batch system ID: 44 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:59:58-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-cev_msom. +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cev_msom +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cev_msom +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-cev_msom follows: +=========> + [2020-10-20T19:59:58-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:58-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:58-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:58-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cev_msom with ID kind-run_mutect_perchrom/instance-cev_msom to 1 +[2020-10-20T19:59:58-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cev_msom with job batch system ID: 45 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:59:58-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpab6zbvec/worker_log.txt +[2020-10-20T19:59:58-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpojcdezzk/worker_log.txt +[2020-10-20T19:59:59-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-gl_w17lv. +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gl_w17lv +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gl_w17lv +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-gl_w17lv follows: +=========> + [2020-10-20T19:59:58-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:58-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:58-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gl_w17lv with ID kind-run_mutect_perchrom/instance-gl_w17lv to 0 +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gl_w17lv with ID kind-run_mutect_perchrom/instance-gl_w17lv is completely failed +[2020-10-20T19:59:59-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-bm4dpfik. +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bm4dpfik +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bm4dpfik +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-bm4dpfik follows: +=========> + [2020-10-20T19:59:58-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:58-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:58-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bm4dpfik with ID kind-run_mutect_perchrom/instance-bm4dpfik to 1 +[2020-10-20T19:59:59-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bm4dpfik with job batch system ID: 46 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:59:59-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpy02q8oxb/worker_log.txt +[2020-10-20T19:59:59-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpktcox5ss/worker_log.txt +[2020-10-20T19:59:59-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-cije5tr0. +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cije5tr0 +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cije5tr0 +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-cije5tr0 follows: +=========> + [2020-10-20T19:59:59-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:59-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:59-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cije5tr0 with ID kind-run_mutect_perchrom/instance-cije5tr0 to 1 +[2020-10-20T19:59:59-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cije5tr0 with job batch system ID: 47 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:59:59-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-40bfs1db. +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-40bfs1db +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-40bfs1db +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-40bfs1db follows: +=========> + [2020-10-20T19:59:59-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:59-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:59-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-40bfs1db with ID kind-run_mutect_perchrom/instance-40bfs1db to 1 +[2020-10-20T19:59:59-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-40bfs1db with job batch system ID: 48 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T19:59:59-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmppgbecg02/worker_log.txt +[2020-10-20T19:59:59-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp7zbjx70s/worker_log.txt +[2020-10-20T19:59:59-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-cev_msom. +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cev_msom +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cev_msom +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-cev_msom follows: +=========> + [2020-10-20T19:59:59-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:59-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:59-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cev_msom with ID kind-run_mutect_perchrom/instance-cev_msom to 0 +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cev_msom with ID kind-run_mutect_perchrom/instance-cev_msom is completely failed +[2020-10-20T19:59:59-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-0oyi7ie_. +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0oyi7ie_ +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0oyi7ie_ +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-0oyi7ie_ follows: +=========> + [2020-10-20T19:59:59-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T19:59:59-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T19:59:59-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0oyi7ie_ with ID kind-run_mutect_perchrom/instance-0oyi7ie_ to 0 +[2020-10-20T19:59:59-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0oyi7ie_ with ID kind-run_mutect_perchrom/instance-0oyi7ie_ is completely failed +[2020-10-20T20:00:00-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp22lvx68s/worker_log.txt +[2020-10-20T20:00:00-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpeg94j_5q/worker_log.txt +[2020-10-20T20:00:00-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-0t3dxg78. +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0t3dxg78 +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0t3dxg78 +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-0t3dxg78 follows: +=========> + [2020-10-20T20:00:00-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:00-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:00-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0t3dxg78 with ID kind-run_mutect_perchrom/instance-0t3dxg78 to 0 +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0t3dxg78 with ID kind-run_mutect_perchrom/instance-0t3dxg78 is completely failed +[2020-10-20T20:00:00-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-6clikwgf. +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-6clikwgf +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-6clikwgf +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-6clikwgf follows: +=========> + [2020-10-20T20:00:00-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:00-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:00-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-6clikwgf with ID kind-run_mutect_perchrom/instance-6clikwgf to 1 +[2020-10-20T20:00:00-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-6clikwgf with job batch system ID: 49 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:00:00-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpigmwmx3j/worker_log.txt +[2020-10-20T20:00:00-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp9t5utgiw/worker_log.txt +[2020-10-20T20:00:00-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-0p9602y6. +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0p9602y6 +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0p9602y6 +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-0p9602y6 follows: +=========> + [2020-10-20T20:00:00-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:00-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:00-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0p9602y6 with ID kind-run_mutect_perchrom/instance-0p9602y6 to 0 +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0p9602y6 with ID kind-run_mutect_perchrom/instance-0p9602y6 is completely failed +[2020-10-20T20:00:00-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-ourgk6vm. +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ourgk6vm +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ourgk6vm +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-ourgk6vm follows: +=========> + [2020-10-20T20:00:00-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:00-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:00-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ourgk6vm with ID kind-run_mutect_perchrom/instance-ourgk6vm to 0 +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ourgk6vm with ID kind-run_mutect_perchrom/instance-ourgk6vm is completely failed +[2020-10-20T20:00:00-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmptcmuqes8/worker_log.txt +[2020-10-20T20:00:00-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp9vj_szc8/worker_log.txt +[2020-10-20T20:00:00-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-8gyqgqyx. +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-8gyqgqyx +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-8gyqgqyx +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-8gyqgqyx follows: +=========> + [2020-10-20T20:00:00-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:00-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:00-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-8gyqgqyx with ID kind-run_mutect_perchrom/instance-8gyqgqyx to 0 +[2020-10-20T20:00:00-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-cp3zlhzq. +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-8gyqgqyx with ID kind-run_mutect_perchrom/instance-8gyqgqyx is completely failed +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cp3zlhzq +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cp3zlhzq +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-cp3zlhzq follows: +=========> + [2020-10-20T20:00:00-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:00-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:00-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cp3zlhzq with ID kind-run_mutect_perchrom/instance-cp3zlhzq to 0 +[2020-10-20T20:00:00-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cp3zlhzq with ID kind-run_mutect_perchrom/instance-cp3zlhzq is completely failed +[2020-10-20T20:00:01-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmph5nse9nj/worker_log.txt +[2020-10-20T20:00:01-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpmr5dn890/worker_log.txt +[2020-10-20T20:00:01-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-vodde7jc. +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-vodde7jc +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-vodde7jc +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-vodde7jc follows: +=========> + [2020-10-20T20:00:01-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:01-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:01-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-vodde7jc with ID kind-run_mutect_perchrom/instance-vodde7jc to 0 +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-vodde7jc with ID kind-run_mutect_perchrom/instance-vodde7jc is completely failed +[2020-10-20T20:00:01-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-h18ep_9e. +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h18ep_9e +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h18ep_9e +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-h18ep_9e follows: +=========> + [2020-10-20T20:00:01-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:01-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:01-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h18ep_9e with ID kind-run_mutect_perchrom/instance-h18ep_9e to 1 +[2020-10-20T20:00:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h18ep_9e with job batch system ID: 50 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:00:01-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmphprc8weq/worker_log.txt +[2020-10-20T20:00:01-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-tjkov9ea. +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-tjkov9ea +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-tjkov9ea +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-tjkov9ea follows: +=========> + [2020-10-20T20:00:01-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:01-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:01-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-tjkov9ea with ID kind-run_mutect_perchrom/instance-tjkov9ea to 1 +[2020-10-20T20:00:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-tjkov9ea with job batch system ID: 51 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:00:01-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpbj0gnit_/worker_log.txt +[2020-10-20T20:00:01-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-wflniclq. +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-wflniclq +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-wflniclq +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-wflniclq follows: +=========> + [2020-10-20T20:00:01-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:01-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:01-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-wflniclq with ID kind-run_mutect_perchrom/instance-wflniclq to 1 +[2020-10-20T20:00:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-wflniclq with job batch system ID: 52 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:00:01-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpn2awtt9m/worker_log.txt +[2020-10-20T20:00:01-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-3a6rn9f9. +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-3a6rn9f9 +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-3a6rn9f9 +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-3a6rn9f9 follows: +=========> + [2020-10-20T20:00:01-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:01-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:01-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:01-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-3a6rn9f9 with ID kind-run_mutect_perchrom/instance-3a6rn9f9 to 1 +[2020-10-20T20:00:01-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-3a6rn9f9 with job batch system ID: 53 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:00:01-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpjbei7ntc/worker_log.txt +[2020-10-20T20:00:02-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-h18ep_9e. +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h18ep_9e +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h18ep_9e +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-h18ep_9e follows: +=========> + [2020-10-20T20:00:01-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:01-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:01-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h18ep_9e with ID kind-run_mutect_perchrom/instance-h18ep_9e to 0 +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h18ep_9e with ID kind-run_mutect_perchrom/instance-h18ep_9e is completely failed +[2020-10-20T20:00:02-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp_1p3hy6k/worker_log.txt +[2020-10-20T20:00:02-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-bm4dpfik. +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bm4dpfik +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bm4dpfik +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-bm4dpfik follows: +=========> + [2020-10-20T20:00:02-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:02-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:02-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bm4dpfik with ID kind-run_mutect_perchrom/instance-bm4dpfik to 0 +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bm4dpfik with ID kind-run_mutect_perchrom/instance-bm4dpfik is completely failed +[2020-10-20T20:00:02-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpt0dk20jm/worker_log.txt +[2020-10-20T20:00:02-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-6clikwgf. +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-6clikwgf +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-6clikwgf +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-6clikwgf follows: +=========> + [2020-10-20T20:00:02-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:02-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:02-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-6clikwgf with ID kind-run_mutect_perchrom/instance-6clikwgf to 0 +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-6clikwgf with ID kind-run_mutect_perchrom/instance-6clikwgf is completely failed +[2020-10-20T20:00:02-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp06cwdrq5/worker_log.txt +[2020-10-20T20:00:02-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-daisoolw. +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-daisoolw +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-daisoolw +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-daisoolw follows: +=========> + [2020-10-20T20:00:02-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:02-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:02-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-daisoolw with ID kind-run_mutect_perchrom/instance-daisoolw to 0 +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-daisoolw with ID kind-run_mutect_perchrom/instance-daisoolw is completely failed +[2020-10-20T20:00:02-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmplpdhisra/worker_log.txt +[2020-10-20T20:00:02-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-9hdk8gj3. +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9hdk8gj3 +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9hdk8gj3 +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-9hdk8gj3 follows: +=========> + [2020-10-20T20:00:02-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:02-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:02-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9hdk8gj3 with ID kind-run_mutect_perchrom/instance-9hdk8gj3 to 0 +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9hdk8gj3 with ID kind-run_mutect_perchrom/instance-9hdk8gj3 is completely failed +[2020-10-20T20:00:02-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp4c5y_3ez/worker_log.txt +[2020-10-20T20:00:02-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-3a6rn9f9. +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-3a6rn9f9 +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-3a6rn9f9 +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-3a6rn9f9 follows: +=========> + [2020-10-20T20:00:02-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:02-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:02-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-3a6rn9f9 with ID kind-run_mutect_perchrom/instance-3a6rn9f9 to 0 +[2020-10-20T20:00:02-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-3a6rn9f9 with ID kind-run_mutect_perchrom/instance-3a6rn9f9 is completely failed +[2020-10-20T20:00:02-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmplzzq842k/worker_log.txt +[2020-10-20T20:00:03-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-92gnpm9m. +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-92gnpm9m +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-92gnpm9m +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-92gnpm9m follows: +=========> + [2020-10-20T20:00:02-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:02-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:02-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-92gnpm9m with ID kind-run_mutect_perchrom/instance-92gnpm9m to 0 +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-92gnpm9m with ID kind-run_mutect_perchrom/instance-92gnpm9m is completely failed +[2020-10-20T20:00:03-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp3gn2clpx/worker_log.txt +[2020-10-20T20:00:03-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-wflniclq. +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-wflniclq +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-wflniclq +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-wflniclq follows: +=========> + [2020-10-20T20:00:03-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:03-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:03-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-wflniclq with ID kind-run_mutect_perchrom/instance-wflniclq to 0 +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-wflniclq with ID kind-run_mutect_perchrom/instance-wflniclq is completely failed +[2020-10-20T20:00:03-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp4r82gt2j/worker_log.txt +[2020-10-20T20:00:03-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-yrvj0gxx. +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yrvj0gxx +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yrvj0gxx +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-yrvj0gxx follows: +=========> + [2020-10-20T20:00:03-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:03-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:03-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yrvj0gxx with ID kind-run_mutect_perchrom/instance-yrvj0gxx to 1 +[2020-10-20T20:00:03-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yrvj0gxx with job batch system ID: 54 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:00:03-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpa4inzpl6/worker_log.txt +[2020-10-20T20:00:03-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-tjkov9ea. +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-tjkov9ea +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-tjkov9ea +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-tjkov9ea follows: +=========> + [2020-10-20T20:00:03-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:03-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:03-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-tjkov9ea with ID kind-run_mutect_perchrom/instance-tjkov9ea to 0 +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-tjkov9ea with ID kind-run_mutect_perchrom/instance-tjkov9ea is completely failed +[2020-10-20T20:00:03-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpi5gyj9pp/worker_log.txt +[2020-10-20T20:00:03-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-40bfs1db. +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-40bfs1db +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-40bfs1db +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-40bfs1db follows: +=========> + [2020-10-20T20:00:03-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:03-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:03-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-40bfs1db with ID kind-run_mutect_perchrom/instance-40bfs1db to 0 +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-40bfs1db with ID kind-run_mutect_perchrom/instance-40bfs1db is completely failed +[2020-10-20T20:00:03-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpmqx7un80/worker_log.txt +[2020-10-20T20:00:03-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-c65720sq. +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-c65720sq +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-c65720sq +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-c65720sq follows: +=========> + [2020-10-20T20:00:03-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:03-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:03-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-c65720sq with ID kind-run_mutect_perchrom/instance-c65720sq to 0 +[2020-10-20T20:00:03-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-c65720sq with ID kind-run_mutect_perchrom/instance-c65720sq is completely failed +[2020-10-20T20:00:03-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp6ngmji95/worker_log.txt +[2020-10-20T20:00:04-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-fu63ea_u. +[2020-10-20T20:00:04-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fu63ea_u +[2020-10-20T20:00:04-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fu63ea_u +[2020-10-20T20:00:04-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-fu63ea_u follows: +=========> + [2020-10-20T20:00:03-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:03-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:04-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:04-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fu63ea_u with ID kind-run_mutect_perchrom/instance-fu63ea_u to 0 +[2020-10-20T20:00:04-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fu63ea_u with ID kind-run_mutect_perchrom/instance-fu63ea_u is completely failed +[2020-10-20T20:00:04-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp3a4po_6d/worker_log.txt +[2020-10-20T20:00:04-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-cije5tr0. +[2020-10-20T20:00:04-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cije5tr0 +[2020-10-20T20:00:04-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cije5tr0 +[2020-10-20T20:00:04-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-cije5tr0 follows: +=========> + [2020-10-20T20:00:04-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:04-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:04-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:04-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cije5tr0 with ID kind-run_mutect_perchrom/instance-cije5tr0 to 0 +[2020-10-20T20:00:04-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cije5tr0 with ID kind-run_mutect_perchrom/instance-cije5tr0 is completely failed +[2020-10-20T20:00:04-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpvfn2mc8k/worker_log.txt +[2020-10-20T20:00:04-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/home/drkthomp/protect3/jobStore kind-run_mutect_perchrom/instance-yrvj0gxx. +[2020-10-20T20:00:04-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yrvj0gxx +[2020-10-20T20:00:04-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yrvj0gxx +[2020-10-20T20:00:04-0700] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-yrvj0gxx follows: +=========> + [2020-10-20T20:00:04-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:00:04-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 123, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/home/drkthomp/protect3/src/protect/common.py", line 172, in gunzip + assert is_gzipfile(input_gzip_file) + AssertionError + [2020-10-20T20:00:04-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-20T20:00:04-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yrvj0gxx with ID kind-run_mutect_perchrom/instance-yrvj0gxx to 0 +[2020-10-20T20:00:04-0700] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yrvj0gxx with ID kind-run_mutect_perchrom/instance-yrvj0gxx is completely failed +[2020-10-20T20:00:04-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmp1vgmc0qo/worker_log.txt diff --git a/errors/2020-10-20-20-18-18.txt b/errors/2020-10-20-20-18-18.txt new file mode 100644 index 00000000..e69de29b diff --git a/errors/2020-10-20-20-40-18.txt b/errors/2020-10-20-20-40-18.txt new file mode 100644 index 00000000..3646e71a --- /dev/null +++ b/errors/2020-10-20-20-40-18.txt @@ -0,0 +1,115 @@ +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-92gnpm9m +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-vodde7jc +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-9hdk8gj3 +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-55hd3p6y +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ourgk6vm +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-3a6rn9f9 +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-gl_w17lv +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-cije5tr0 +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-cp3zlhzq +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-dyn6bo3z +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-0t3dxg78 +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-wyy831pe +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-tjkov9ea +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-6clikwgf +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-0oyi7ie_ +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-cev_msom +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-h18ep_9e +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-yrvj0gxx +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-daisoolw +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-c65720sq +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-fu63ea_u +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-sfhpnee3 +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-j5t5f3ax +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-bm4dpfik +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-8gyqgqyx +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-40bfs1db +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-wflniclq +[2020-10-20T20:40:21-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-0p9602y6 +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-wyy831pe with job batch system ID: 0 and cores: 4, disk: 7.8 G, and memory: 2.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-j5t5f3ax with job batch system ID: 1 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-8gyqgqyx with job batch system ID: 2 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-55hd3p6y with job batch system ID: 3 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cije5tr0 with job batch system ID: 4 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cp3zlhzq with job batch system ID: 5 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h18ep_9e with job batch system ID: 6 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-vodde7jc with job batch system ID: 7 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-40bfs1db with job batch system ID: 8 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-92gnpm9m with job batch system ID: 9 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gl_w17lv with job batch system ID: 10 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-6clikwgf with job batch system ID: 11 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0p9602y6 with job batch system ID: 12 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-tjkov9ea with job batch system ID: 13 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9hdk8gj3 with job batch system ID: 14 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cev_msom with job batch system ID: 15 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fu63ea_u with job batch system ID: 16 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0oyi7ie_ with job batch system ID: 17 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ourgk6vm with job batch system ID: 18 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-3a6rn9f9 with job batch system ID: 19 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-sfhpnee3 with job batch system ID: 20 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bm4dpfik with job batch system ID: 21 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0t3dxg78 with job batch system ID: 22 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-daisoolw with job batch system ID: 23 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-c65720sq with job batch system ID: 24 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-wflniclq with job batch system ID: 25 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yrvj0gxx with job batch system ID: 26 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-dyn6bo3z with job batch system ID: 27 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.leader] Issued job 'run_star' kind-run_star/instance-5q77j6ix with job batch system ID: 28 and cores: 4, disk: 52.8 G, and memory: 48.6 G +[2020-10-20T20:40:21-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpfjgyriub/worker_log.txt +[2020-10-20T20:40:23-0700] [MainThread] [I] [toil.leader] 1 jobs are running, 28 jobs are issued and waiting to run +[2020-10-20T20:43:43-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/home/drkthomp/protect3/jobStore kind-run_phlat/instance-wyy831pe. +[2020-10-20T20:43:43-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-wyy831pe +[2020-10-20T20:43:43-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-wyy831pe +[2020-10-20T20:43:43-0700] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-wyy831pe follows: +=========> + [2020-10-20T20:40:21-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-20T20:40:21-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + 567573 reads; of these: + 567573 (100.00%) were paired; of these: + 3570 (0.63%) aligned concordantly 0 times + 514680 (90.68%) aligned concordantly exactly 1 time + 49323 (8.69%) aligned concordantly >1 times + 99.37% overall alignment rate + .....Process Bowtie 2 mapping on tumor_dna....... + + .....Prepare files of tumor_dna for PHLAT....... + + ..... Running PHLAT ....... + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 64, in + go(tag,outdir,phlatdir) + File "/home/yubai/programs/phlat-1.0/dist/utilities.py", line 79, in go + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 617, in __init__ + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 645, in setMaster + IndexError: list index out of range + Traceback (most recent call last): + File "/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib/python3.8/subprocess.py", line 364, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpfjgyriub/1fd46caf-509b-4911-9269-b5bb6817c571:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '4']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 73, in run_phlat + docker_call(tool='phlat', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 138, in docker_call + raise RuntimeError('docker command returned a non-zero exit status (%s)' % err.returncode + + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpfjgyriub/1fd46caf-509b-4911-9269-b5bb6817c571:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_dna -e /home/phlat-1.0 -o /data -p 4" + [2020-10-20T20:43:42-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox + ['-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '4'] +<========= +[2020-10-20T20:43:43-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-wyy831pe with ID kind-run_phlat/instance-wyy831pe to 1 +[2020-10-20T20:43:43-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-wyy831pe with job batch system ID: 29 and cores: 4, disk: 7.8 G, and memory: 2.0 G +[2020-10-20T20:43:44-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpw0rb1_a_/worker_log.txt +[2020-10-20T20:43:44-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmpw1fve6r6/worker_log.txt diff --git a/errors/2020-10-22-17-18-33.txt b/errors/2020-10-22-17-18-33.txt new file mode 100755 index 00000000..b6b0b32c --- /dev/null +++ b/errors/2020-10-22-17-18-33.txt @@ -0,0 +1,33 @@ +[2020-10-22T17:18:39-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-wyy831pe +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-wyy831pe with job batch system ID: 0 and cores: 4, disk: 7.8 G, and memory: 2.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-j5t5f3ax with job batch system ID: 1 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-8gyqgqyx with job batch system ID: 2 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-55hd3p6y with job batch system ID: 3 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cije5tr0 with job batch system ID: 4 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cp3zlhzq with job batch system ID: 5 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h18ep_9e with job batch system ID: 6 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-vodde7jc with job batch system ID: 7 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-40bfs1db with job batch system ID: 8 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-92gnpm9m with job batch system ID: 9 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gl_w17lv with job batch system ID: 10 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-6clikwgf with job batch system ID: 11 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0p9602y6 with job batch system ID: 12 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-tjkov9ea with job batch system ID: 13 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9hdk8gj3 with job batch system ID: 14 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-cev_msom with job batch system ID: 15 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fu63ea_u with job batch system ID: 16 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0oyi7ie_ with job batch system ID: 17 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ourgk6vm with job batch system ID: 18 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-3a6rn9f9 with job batch system ID: 19 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-sfhpnee3 with job batch system ID: 20 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bm4dpfik with job batch system ID: 21 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0t3dxg78 with job batch system ID: 22 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-daisoolw with job batch system ID: 23 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-c65720sq with job batch system ID: 24 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-wflniclq with job batch system ID: 25 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yrvj0gxx with job batch system ID: 26 and cores: 1, disk: 23.0 G, and memory: 6.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-dyn6bo3z with job batch system ID: 27 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-22T17:18:39-0700] [MainThread] [I] [toil.leader] Issued job 'run_star' kind-run_star/instance-5q77j6ix with job batch system ID: 28 and cores: 4, disk: 52.8 G, and memory: 48.6 G +[2020-10-22T17:18:40-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-72996abd-6cf4-462a-aea1-4d483640105b-1e53f69f16d74e0b881e7db602b2d2c3/tmposqjat6w/worker_log.txt +[2020-10-22T17:18:41-0700] [MainThread] [I] [toil.leader] 1 jobs are running, 28 jobs are issued and waiting to run diff --git a/errors/2020-10-22-17-19-05.txt b/errors/2020-10-22-17-19-05.txt new file mode 100755 index 00000000..e5ccba59 --- /dev/null +++ b/errors/2020-10-22-17-19-05.txt @@ -0,0 +1,12 @@ +Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/bin/ProTECT", line 33, in + sys.exit(load_entry_point('protect', 'console_scripts', 'ProTECT')()) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1026, in main + Job.Runner.startToil(start, params) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 794, in startToil + with Toil(options) as toil: + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/common.py", line 776, in __enter__ + jobStore.resume() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/jobStores/fileJobStore.py", line 121, in resume + raise NoSuchJobStoreException(self.jobStoreDir) +toil.jobStores.abstractJobStore.NoSuchJobStoreException: The job store '/home/drkthomp/protect3/jobStore' does not exist, so there is nothing to restart. diff --git a/errors/2020-10-22-17-19-15.txt b/errors/2020-10-22-17-19-15.txt new file mode 100755 index 00000000..e5ccba59 --- /dev/null +++ b/errors/2020-10-22-17-19-15.txt @@ -0,0 +1,12 @@ +Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/bin/ProTECT", line 33, in + sys.exit(load_entry_point('protect', 'console_scripts', 'ProTECT')()) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1026, in main + Job.Runner.startToil(start, params) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 794, in startToil + with Toil(options) as toil: + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/common.py", line 776, in __enter__ + jobStore.resume() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/jobStores/fileJobStore.py", line 121, in resume + raise NoSuchJobStoreException(self.jobStoreDir) +toil.jobStores.abstractJobStore.NoSuchJobStoreException: The job store '/home/drkthomp/protect3/jobStore' does not exist, so there is nothing to restart. diff --git a/errors/2020-10-22-17-19-23.txt b/errors/2020-10-22-17-19-23.txt new file mode 100755 index 00000000..f9380b5c --- /dev/null +++ b/errors/2020-10-22-17-19-23.txt @@ -0,0 +1,121 @@ +[2020-10-22T17:19:23-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. +[2020-10-22T17:19:23-0700] [MainThread] [I] [toil.leader] Issued job 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 with job batch system ID: 0 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-22T17:19:23-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmpsa5trpov/worker_log.txt +[2020-10-22T17:19:24-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker parse_config_file file:/home/drkthomp/protect3/jobStore kind-parse_config_file/instance-s_nr25h6. +[2020-10-22T17:19:24-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 +[2020-10-22T17:19:24-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 +[2020-10-22T17:19:24-0700] [MainThread] [W] [toil.leader] Log from job kind-parse_config_file/instance-s_nr25h6 follows: +=========> + [2020-10-22T17:19:23-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-22T17:19:23-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + [2020-10-22T17:19:23-0700] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Parsing config file + /home/drkthomp/protect3/src/protect/pipeline/ProTECT.py:334: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details. + input_config = yaml.load(conf.read()) + /home/drkthomp/protect3/src/protect/pipeline/ProTECT.py:339: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details. + required_keys = yaml.load(ref.read()) + /home/drkthomp/protect3/src/protect/pipeline/ProTECT.py:351: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details. + protect_defaults = yaml.load(pdf.read()) + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 444, in parse_config_file + sample_set, univ_options, processed_tool_inputs = _parse_config_file(job, config_file, + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 363, in _parse_config_file + _ensure_patient_group_is_ok(patient_keys, sample_name) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 120, in _ensure_patient_group_is_ok + raise ParameterError(('The patient entry for sample %s ' % patient_name) + + protect.common.ParameterError: The patient entry for sample TEST does not contain a hla_haplotype_files entry. + Cannot haplotype patient if all the input sequence files are not fastqs. + [2020-10-22T17:19:23-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-22T17:19:24-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 with ID kind-parse_config_file/instance-s_nr25h6 to 1 +[2020-10-22T17:19:24-0700] [MainThread] [I] [toil.leader] 0 jobs are running, 0 jobs are issued and waiting to run +[2020-10-22T17:19:24-0700] [MainThread] [I] [toil.leader] Issued job 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 with job batch system ID: 1 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-22T17:19:24-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmpyy3gjlcr/worker_log.txt +[2020-10-22T17:19:24-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker parse_config_file file:/home/drkthomp/protect3/jobStore kind-parse_config_file/instance-s_nr25h6. +[2020-10-22T17:19:24-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 +[2020-10-22T17:19:24-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 +[2020-10-22T17:19:24-0700] [MainThread] [W] [toil.leader] Log from job kind-parse_config_file/instance-s_nr25h6 follows: +=========> + [2020-10-22T17:19:24-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-22T17:19:24-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + [2020-10-22T17:19:24-0700] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Parsing config file + /home/drkthomp/protect3/src/protect/pipeline/ProTECT.py:334: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details. + input_config = yaml.load(conf.read()) + /home/drkthomp/protect3/src/protect/pipeline/ProTECT.py:339: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details. + required_keys = yaml.load(ref.read()) + /home/drkthomp/protect3/src/protect/pipeline/ProTECT.py:351: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details. + protect_defaults = yaml.load(pdf.read()) + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 444, in parse_config_file + sample_set, univ_options, processed_tool_inputs = _parse_config_file(job, config_file, + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 363, in _parse_config_file + _ensure_patient_group_is_ok(patient_keys, sample_name) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 120, in _ensure_patient_group_is_ok + raise ParameterError(('The patient entry for sample %s ' % patient_name) + + protect.common.ParameterError: The patient entry for sample TEST does not contain a hla_haplotype_files entry. + Cannot haplotype patient if all the input sequence files are not fastqs. + [2020-10-22T17:19:24-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-22T17:19:24-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 with ID kind-parse_config_file/instance-s_nr25h6 to 0 +[2020-10-22T17:19:24-0700] [MainThread] [W] [toil.leader] Job 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 with ID kind-parse_config_file/instance-s_nr25h6 is completely failed +[2020-10-22T17:19:28-0700] [MainThread] [I] [toil.leader] Finished toil run with 1 failed jobs. +[2020-10-22T17:19:28-0700] [MainThread] [I] [toil.leader] Failed jobs at end of the run: 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 +Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/bin/ProTECT", line 33, in + sys.exit(load_entry_point('protect', 'console_scripts', 'ProTECT')()) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1026, in main + Job.Runner.startToil(start, params) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 796, in startToil + return toil.start(job) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/common.py", line 846, in start + return self._runMainLoop(rootJobGraph) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/common.py", line 1133, in _runMainLoop + return Leader(config=self.config, + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/leader.py", line 269, in run + raise FailedJobsException(self.config.jobStore, self.toilState.totalFailedJobs, self.jobStore) +toil.leader.FailedJobsException: The job store 'file:/home/drkthomp/protect3/jobStore' contains 1 failed jobs: 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 +Log from job 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 follows: +=========> + [2020-10-22T17:19:24-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-22T17:19:24-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + [2020-10-22T17:19:24-0700] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Parsing config file + /home/drkthomp/protect3/src/protect/pipeline/ProTECT.py:334: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details. + input_config = yaml.load(conf.read()) + /home/drkthomp/protect3/src/protect/pipeline/ProTECT.py:339: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details. + required_keys = yaml.load(ref.read()) + /home/drkthomp/protect3/src/protect/pipeline/ProTECT.py:351: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details. + protect_defaults = yaml.load(pdf.read()) + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 444, in parse_config_file + sample_set, univ_options, processed_tool_inputs = _parse_config_file(job, config_file, + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 363, in _parse_config_file + _ensure_patient_group_is_ok(patient_keys, sample_name) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 120, in _ensure_patient_group_is_ok + raise ParameterError(('The patient entry for sample %s ' % patient_name) + + protect.common.ParameterError: The patient entry for sample TEST does not contain a hla_haplotype_files entry. + Cannot haplotype patient if all the input sequence files are not fastqs. + [2020-10-22T17:19:24-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= diff --git a/errors/2020-10-31-03-14-30.txt b/errors/2020-10-31-03-14-30.txt new file mode 100755 index 00000000..ee5a0601 --- /dev/null +++ b/errors/2020-10-31-03-14-30.txt @@ -0,0 +1,197 @@ +[2020-10-31T03:14:36-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-parse_config_file/instance-s_nr25h6 +[2020-10-31T03:14:36-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. +[2020-10-31T03:14:36-0700] [MainThread] [I] [toil.leader] Issued job 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 with job batch system ID: 0 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:37-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmpb02q6w3q/worker_log.txt +[2020-10-31T03:14:37-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker parse_config_file file:/home/drkthomp/protect3/jobStore kind-parse_config_file/instance-s_nr25h6. +[2020-10-31T03:14:37-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 +[2020-10-31T03:14:37-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 +[2020-10-31T03:14:37-0700] [MainThread] [W] [toil.leader] Log from job kind-parse_config_file/instance-s_nr25h6 follows: +=========> + [2020-10-31T03:14:37-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-31T03:14:37-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + [2020-10-31T03:14:37-0700] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Parsing config file + /home/drkthomp/protect3/src/protect/pipeline/ProTECT.py:333: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details. + input_config = yaml.load(conf.read()) + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 443, in parse_config_file + sample_set, univ_options, processed_tool_inputs = _parse_config_file(job, config_file, + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 333, in _parse_config_file + input_config = yaml.load(conf.read()) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/__init__.py", line 114, in load + return loader.get_single_data() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/constructor.py", line 49, in get_single_data + node = self.get_single_node() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 36, in get_single_node + document = self.compose_document() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 55, in compose_document + node = self.compose_node(None, None) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 84, in compose_node + node = self.compose_mapping_node(anchor) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 133, in compose_mapping_node + item_value = self.compose_node(node, item_key) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 84, in compose_node + node = self.compose_mapping_node(anchor) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 133, in compose_mapping_node + item_value = self.compose_node(node, item_key) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 84, in compose_node + node = self.compose_mapping_node(anchor) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 127, in compose_mapping_node + while not self.check_event(MappingEndEvent): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/parser.py", line 98, in check_event + self.current_event = self.state() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/parser.py", line 428, in parse_block_mapping_key + if self.check_token(KeyToken): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/scanner.py", line 116, in check_token + self.fetch_more_tokens() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/scanner.py", line 258, in fetch_more_tokens + raise ScannerError("while scanning for the next token", None, + yaml.scanner.ScannerError: while scanning for the next token + found character '\t' that cannot start any token + in "", line 32, column 1: + tumor_dna_fastq_2: /home/drktho ... + ^ + [2020-10-31T03:14:37-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-31T03:14:37-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 with ID kind-parse_config_file/instance-s_nr25h6 to 1 +[2020-10-31T03:14:37-0700] [MainThread] [I] [toil.leader] 0 jobs are running, 0 jobs are issued and waiting to run +[2020-10-31T03:14:37-0700] [MainThread] [I] [toil.leader] Issued job 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 with job batch system ID: 1 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:37-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmp0l1bqexv/worker_log.txt +[2020-10-31T03:14:37-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker parse_config_file file:/home/drkthomp/protect3/jobStore kind-parse_config_file/instance-s_nr25h6. +[2020-10-31T03:14:37-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 +[2020-10-31T03:14:37-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 +[2020-10-31T03:14:37-0700] [MainThread] [W] [toil.leader] Log from job kind-parse_config_file/instance-s_nr25h6 follows: +=========> + [2020-10-31T03:14:37-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-31T03:14:37-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + [2020-10-31T03:14:37-0700] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Parsing config file + /home/drkthomp/protect3/src/protect/pipeline/ProTECT.py:333: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details. + input_config = yaml.load(conf.read()) + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 443, in parse_config_file + sample_set, univ_options, processed_tool_inputs = _parse_config_file(job, config_file, + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 333, in _parse_config_file + input_config = yaml.load(conf.read()) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/__init__.py", line 114, in load + return loader.get_single_data() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/constructor.py", line 49, in get_single_data + node = self.get_single_node() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 36, in get_single_node + document = self.compose_document() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 55, in compose_document + node = self.compose_node(None, None) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 84, in compose_node + node = self.compose_mapping_node(anchor) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 133, in compose_mapping_node + item_value = self.compose_node(node, item_key) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 84, in compose_node + node = self.compose_mapping_node(anchor) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 133, in compose_mapping_node + item_value = self.compose_node(node, item_key) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 84, in compose_node + node = self.compose_mapping_node(anchor) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 127, in compose_mapping_node + while not self.check_event(MappingEndEvent): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/parser.py", line 98, in check_event + self.current_event = self.state() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/parser.py", line 428, in parse_block_mapping_key + if self.check_token(KeyToken): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/scanner.py", line 116, in check_token + self.fetch_more_tokens() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/scanner.py", line 258, in fetch_more_tokens + raise ScannerError("while scanning for the next token", None, + yaml.scanner.ScannerError: while scanning for the next token + found character '\t' that cannot start any token + in "", line 32, column 1: + tumor_dna_fastq_2: /home/drktho ... + ^ + [2020-10-31T03:14:37-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-31T03:14:37-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 with ID kind-parse_config_file/instance-s_nr25h6 to 0 +[2020-10-31T03:14:37-0700] [MainThread] [W] [toil.leader] Job 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 with ID kind-parse_config_file/instance-s_nr25h6 is completely failed +[2020-10-31T03:14:41-0700] [MainThread] [I] [toil.leader] Finished toil run with 1 failed jobs. +[2020-10-31T03:14:41-0700] [MainThread] [I] [toil.leader] Failed jobs at end of the run: 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 +Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/bin/ProTECT", line 33, in + sys.exit(load_entry_point('protect', 'console_scripts', 'ProTECT')()) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1025, in main + Job.Runner.startToil(start, params) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 798, in startToil + return toil.restart() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/common.py", line 878, in restart + return self._runMainLoop(rootJobGraph) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/common.py", line 1133, in _runMainLoop + return Leader(config=self.config, + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/leader.py", line 269, in run + raise FailedJobsException(self.config.jobStore, self.toilState.totalFailedJobs, self.jobStore) +toil.leader.FailedJobsException: The job store 'file:/home/drkthomp/protect3/jobStore' contains 1 failed jobs: 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 +Log from job 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 follows: +=========> + [2020-10-31T03:14:37-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-31T03:14:37-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + [2020-10-31T03:14:37-0700] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Parsing config file + /home/drkthomp/protect3/src/protect/pipeline/ProTECT.py:333: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details. + input_config = yaml.load(conf.read()) + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 443, in parse_config_file + sample_set, univ_options, processed_tool_inputs = _parse_config_file(job, config_file, + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 333, in _parse_config_file + input_config = yaml.load(conf.read()) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/__init__.py", line 114, in load + return loader.get_single_data() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/constructor.py", line 49, in get_single_data + node = self.get_single_node() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 36, in get_single_node + document = self.compose_document() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 55, in compose_document + node = self.compose_node(None, None) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 84, in compose_node + node = self.compose_mapping_node(anchor) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 133, in compose_mapping_node + item_value = self.compose_node(node, item_key) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 84, in compose_node + node = self.compose_mapping_node(anchor) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 133, in compose_mapping_node + item_value = self.compose_node(node, item_key) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 84, in compose_node + node = self.compose_mapping_node(anchor) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/composer.py", line 127, in compose_mapping_node + while not self.check_event(MappingEndEvent): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/parser.py", line 98, in check_event + self.current_event = self.state() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/parser.py", line 428, in parse_block_mapping_key + if self.check_token(KeyToken): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/scanner.py", line 116, in check_token + self.fetch_more_tokens() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/yaml/scanner.py", line 258, in fetch_more_tokens + raise ScannerError("while scanning for the next token", None, + yaml.scanner.ScannerError: while scanning for the next token + found character '\t' that cannot start any token + in "", line 32, column 1: + tumor_dna_fastq_2: /home/drktho ... + ^ + [2020-10-31T03:14:37-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= diff --git a/errors/2020-10-31-03-14-53.txt b/errors/2020-10-31-03-14-53.txt new file mode 100755 index 00000000..1253c922 --- /dev/null +++ b/errors/2020-10-31-03-14-53.txt @@ -0,0 +1,263 @@ +[2020-10-31T03:14:53-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-parse_config_file/instance-s_nr25h6 +[2020-10-31T03:14:53-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. +[2020-10-31T03:14:53-0700] [MainThread] [I] [toil.leader] Issued job 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 with job batch system ID: 0 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:53-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmp5xaxxgz_/worker_log.txt +[2020-10-31T03:14:54-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:14:54: Parsing config file +[2020-10-31T03:14:54-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:14:54: Obtaining tool inputs +[2020-10-31T03:14:54-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:14:54: Obtained tool inputs +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Job ended: 'parse_config_file' kind-parse_config_file/instance-s_nr25h6 +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] 0 jobs are running, 0 jobs are issued and waiting to run +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ol5yragf with job batch system ID: 1 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-bnqhiowx with job batch system ID: 2 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-23ubwjg4 with job batch system ID: 3 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-a13duj94 with job batch system ID: 4 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-kqbkpqbg with job batch system ID: 5 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-evbljlp_ with job batch system ID: 6 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-0ue_rp4g with job batch system ID: 7 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6xzk3vxr with job batch system ID: 8 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-7dvy2nyz with job batch system ID: 9 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-uxwvzfpa with job batch system ID: 10 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-cv5plrdz with job batch system ID: 11 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-t3hqemvt with job batch system ID: 12 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-3bli8nkm with job batch system ID: 13 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-8ltngf_z with job batch system ID: 14 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-m65crt5w with job batch system ID: 15 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-0stnob69 with job batch system ID: 16 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-9u3qs9s2 with job batch system ID: 17 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5ig6q9qp with job batch system ID: 18 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ukj421y7 with job batch system ID: 19 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-m40jw59g with job batch system ID: 20 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-s4038s7p with job batch system ID: 21 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6evawovr with job batch system ID: 22 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-lc57ilkz with job batch system ID: 23 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ir95xezi with job batch system ID: 24 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5fu379ei with job batch system ID: 25 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-n7brxvqs with job batch system ID: 26 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-j33lpr99 with job batch system ID: 27 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-r8q7_wes with job batch system ID: 28 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-p7fjj4sc with job batch system ID: 29 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmp5yxlzuet/worker_log.txt +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmp8b7l96xo/worker_log.txt +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmpycjrgogp/worker_log.txt +[2020-10-31T03:14:54-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmph03_0k2v/worker_log.txt +[2020-10-31T03:24:29-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-a13duj94 +[2020-10-31T03:24:29-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:24:29: Obtaining file (indexes:genome_fasta) to the file job store +[2020-10-31T03:24:30-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmpwgqdg01v/worker_log.txt +[2020-10-31T03:24:30-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-j33lpr99 +[2020-10-31T03:24:30-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmp70xr90_p/worker_log.txt +[2020-10-31T03:24:30-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:24:30: Obtaining file (reports:itx_resistance_file) to the file job store +[2020-10-31T03:24:31-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-kqbkpqbg +[2020-10-31T03:24:31-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmpypoowjie/worker_log.txt +[2020-10-31T03:24:31-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:24:31: Obtaining file (indexes:genome_fai) to the file job store +[2020-10-31T03:26:00-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:26:00: Obtaining file (rsem:index) to the file job store +[2020-10-31T03:26:00-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-23ubwjg4 +[2020-10-31T03:26:00-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmp9ce0ms3d/worker_log.txt +[2020-10-31T03:35:22-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6evawovr +[2020-10-31T03:35:22-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:35:22: Obtaining file (transgene:genome_fasta) to the file job store +[2020-10-31T03:35:23-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmpapws6tv8/worker_log.txt +[2020-10-31T03:35:24-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-cv5plrdz +[2020-10-31T03:35:25-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmp7hi77spj/worker_log.txt +[2020-10-31T03:35:25-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:35:25: Obtaining file (indexes:dbsnp_tbi) to the file job store +[2020-10-31T03:45:00-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-3bli8nkm +[2020-10-31T03:45:00-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:45:00: Obtaining file (radia:dbsnp_beds) to the file job store +[2020-10-31T03:45:01-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmpz6zz9lx5/worker_log.txt +[2020-10-31T03:45:02-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-7dvy2nyz +[2020-10-31T03:45:02-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-0stnob69 +[2020-10-31T03:45:02-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmp418u1p9c/worker_log.txt +[2020-10-31T03:45:02-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmpvzzd3w70/worker_log.txt +[2020-10-31T03:45:03-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:45:03: Obtaining file (indexes:dbsnp_vcf) to the file job store +[2020-10-31T03:45:03-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:45:03: Obtaining file (radia:gencode_beds) to the file job store +[2020-10-31T03:45:03-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:45:03: Obtaining file (indexes:cosmic_idx) to the file job store +[2020-10-31T03:45:03-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6xzk3vxr +[2020-10-31T03:45:03-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmpwwkf8pgz/worker_log.txt +[2020-10-31T03:45:04-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5fu379ei +[2020-10-31T03:45:04-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:45:04: Obtaining file (mhcii:method_file) to the file job store +[2020-10-31T03:45:04-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmpcyz386_w/worker_log.txt +[2020-10-31T03:45:19-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-t3hqemvt +[2020-10-31T03:45:19-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:45:19: Obtaining file (radia:cosmic_beds) to the file job store +[2020-10-31T03:45:19-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmp_zqxrkct/worker_log.txt +[2020-10-31T03:45:20-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:45:20: Obtaining file (strelka:config_file) to the file job store +[2020-10-31T03:45:20-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-9u3qs9s2 +[2020-10-31T03:45:20-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmp5ds8j_z0/worker_log.txt +[2020-10-31T03:45:24-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-m40jw59g +[2020-10-31T03:45:25-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmpno8vvwls/worker_log.txt +[2020-10-31T03:45:25-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:45:25: Obtaining file (transgene:gencode_transcript_fasta) to the file job store +[2020-10-31T03:45:27-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ukj421y7 +[2020-10-31T03:45:27-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmpf2mfhe53/worker_log.txt +[2020-10-31T03:45:27-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:45:27: Obtaining file (transgene:gencode_peptide_fasta) to the file job store +[2020-10-31T03:45:27-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-8ltngf_z +[2020-10-31T03:45:28-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmp5f7oxj0y/worker_log.txt +[2020-10-31T03:45:28-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:45:28: Obtaining file (radia:retrogene_beds) to the file job store +[2020-10-31T03:45:28-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-m65crt5w +[2020-10-31T03:45:28-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:45:28: Obtaining file (radia:pseudogene_beds) to the file job store +[2020-10-31T03:45:28-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmphkscq23i/worker_log.txt +[2020-10-31T03:45:29-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-evbljlp_ +[2020-10-31T03:45:29-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:45:29: Obtaining file (indexes:genome_dict) to the file job store +[2020-10-31T03:45:29-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmpbd42lewz/worker_log.txt +[2020-10-31T03:45:58-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-0ue_rp4g +[2020-10-31T03:45:58-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:45:58: Obtaining file (indexes:cosmic_vcf) to the file job store +[2020-10-31T03:45:58-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmppgxjvld2/worker_log.txt +[2020-10-31T03:45:59-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-p7fjj4sc +[2020-10-31T03:45:59-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:45:59: Obtaining file (reports:car_t_targets_file) to the file job store +[2020-10-31T03:45:59-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmpf7w2rnjz/worker_log.txt +[2020-10-31T03:46:01-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-uxwvzfpa +[2020-10-31T03:46:01-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmp678zwdd4/worker_log.txt +[2020-10-31T03:46:01-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:46:01: Obtaining file (indexes:dbsnp_idx) to the file job store +[2020-10-31T03:46:01-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ir95xezi +[2020-10-31T03:46:02-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmp5bid24om/worker_log.txt +[2020-10-31T03:46:02-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:46:02: Obtaining file (mhci:method_file) to the file job store +[2020-10-31T03:46:02-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-r8q7_wes +[2020-10-31T03:46:02-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmp0x3ewm91/worker_log.txt +[2020-10-31T03:46:02-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:46:02: Obtaining file (reports:immune_resistance_pathways_file) to the file job store +[2020-10-31T03:46:02-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-n7brxvqs +[2020-10-31T03:46:03-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmp4_19j57j/worker_log.txt +[2020-10-31T03:46:03-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:46:03: Obtaining file (reports:mhc_pathways_file) to the file job store +[2020-10-31T03:46:29-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-s4038s7p +[2020-10-31T03:46:29-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmpo1m3b5z_/worker_log.txt +[2020-10-31T03:46:29-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:46:29: Obtaining file (transgene:gencode_annotation_gtf) to the file job store +[2020-10-31T03:49:10-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-bnqhiowx +[2020-10-31T03:49:10-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:49:10: Obtaining file (bwa:index) to the file job store +[2020-10-31T03:55:12-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5ig6q9qp +[2020-10-31T03:55:13-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 03:55:13: Obtaining file (snpeff:index) to the file job store +[2020-10-31T04:07:52-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-lc57ilkz +[2020-10-31T04:07:52-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 04:07:52: Obtaining file (phlat:index) to the file job store +[2020-10-31T04:14:54-0700] [MainThread] [I] [toil.leader] 1 jobs are running, 0 jobs are issued and waiting to run +[2020-10-31T04:27:35-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ol5yragf +[2020-10-31T04:27:35-0700] [MainThread] [I] [toil.leader] Issued job 'launch_protect' kind-launch_protect/instance-_8136hgg with job batch system ID: 30 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-31T04:27:35-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-31-2020 04:27:35: Obtaining file (star:index) to the file job store +[2020-10-31T04:27:37-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmp2o3vdmzs/worker_log.txt +[2020-10-31T04:27:38-0700] [MainThread] [I] [toil.leader] Job ended: 'launch_protect' kind-launch_protect/instance-_8136hgg +[2020-10-31T04:27:38-0700] [MainThread] [I] [toil.leader] Issued job 'prepare_samples' kind-prepare_samples/instance-ut9mbtly with job batch system ID: 31 and cores: 1, disk: 40.0 G, and memory: 2.0 G +[2020-10-31T04:27:38-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmpw1os94d1/worker_log.txt +[2020-10-31T04:34:43-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker prepare_samples file:/home/drkthomp/protect3/jobStore kind-prepare_samples/instance-ut9mbtly. +[2020-10-31T04:34:43-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'prepare_samples' kind-prepare_samples/instance-ut9mbtly +[2020-10-31T04:34:43-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'prepare_samples' kind-prepare_samples/instance-ut9mbtly +[2020-10-31T04:34:43-0700] [MainThread] [W] [toil.leader] Log from job kind-prepare_samples/instance-ut9mbtly follows: +=========> + [2020-10-31T04:27:38-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-31T04:27:38-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + [2020-10-31T04:27:38-0700] [MainThread] [W] [toil.fileStores.nonCachingFileStore] Starting job files/for-job/kind-launch_protect/instance-_8136hgg/cleanup/file-rbhwf616/stream with less than 10% of disk space remaining. + [2020-10-31T04:27:38-0700] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Downloading Inputs for TEST + [2020-10-31T04:27:38-0700] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (TEST:tumor_dna_fastq_1) to the file job store + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 840, in prepare_samples + output_dict[input_file] = get_pipeline_inputs( + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 805, in get_pipeline_inputs + input_file = job.fileStore.writeGlobalFile(input_file) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/fileStores/nonCachingFileStore.py", line 91, in writeGlobalFile + fileStoreID = self.jobStore.writeFile(absLocalFileName, creatorID, cleanup) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/jobStores/fileJobStore.py", line 387, in writeFile + atomic_copy(localFilePath, absPath) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/lib/misc.py", line 225, in atomic_copy + shutil.copyfile(src_path, dest_path_tmp) + File "/usr/lib/python3.8/shutil.py", line 272, in copyfile + _fastcopy_sendfile(fsrc, fdst) + File "/usr/lib/python3.8/shutil.py", line 163, in _fastcopy_sendfile + raise err from None + File "/usr/lib/python3.8/shutil.py", line 149, in _fastcopy_sendfile + sent = os.sendfile(outfd, infd, offset, blocksize) + OSError: [Errno 28] No space left on device: '/home/drkthomp/gerald_D1VCPACXX_1_1.fastq' -> '/home/drkthomp/protect3/jobStore/files/for-job/kind-prepare_samples/instance-ut9mbtly/file-b2fkysiw/gerald_D1VCPACXX_1_1.fastq.95d871c7-21b4-4cbc-86c4-bdc16bb29163.tmp.fastq' + [2020-10-31T04:34:43-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-31T04:34:43-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'prepare_samples' kind-prepare_samples/instance-ut9mbtly with ID kind-prepare_samples/instance-ut9mbtly to 1 +[2020-10-31T04:34:43-0700] [MainThread] [I] [toil.leader] Issued job 'prepare_samples' kind-prepare_samples/instance-ut9mbtly with job batch system ID: 32 and cores: 1, disk: 40.0 G, and memory: 2.0 G +[2020-10-31T04:34:44-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-f1f8997a-f8d2-4f90-ad06-953f95d5bcdd-1e53f69f16d74e0b881e7db602b2d2c3/tmpfuf23bvl/worker_log.txt +[2020-10-31T04:42:49-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker prepare_samples file:/home/drkthomp/protect3/jobStore kind-prepare_samples/instance-ut9mbtly. +[2020-10-31T04:42:49-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'prepare_samples' kind-prepare_samples/instance-ut9mbtly +[2020-10-31T04:42:49-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'prepare_samples' kind-prepare_samples/instance-ut9mbtly +[2020-10-31T04:42:49-0700] [MainThread] [W] [toil.leader] Log from job kind-prepare_samples/instance-ut9mbtly follows: +=========> + [2020-10-31T04:34:44-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-31T04:34:44-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + [2020-10-31T04:34:44-0700] [MainThread] [W] [toil.fileStores.nonCachingFileStore] Starting job files/for-job/kind-launch_protect/instance-_8136hgg/cleanup/file-rbhwf616/stream with less than 10% of disk space remaining. + [2020-10-31T04:34:44-0700] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Downloading Inputs for TEST + [2020-10-31T04:34:44-0700] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (TEST:tumor_dna_fastq_1) to the file job store + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 840, in prepare_samples + output_dict[input_file] = get_pipeline_inputs( + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 805, in get_pipeline_inputs + input_file = job.fileStore.writeGlobalFile(input_file) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/fileStores/nonCachingFileStore.py", line 91, in writeGlobalFile + fileStoreID = self.jobStore.writeFile(absLocalFileName, creatorID, cleanup) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/jobStores/fileJobStore.py", line 387, in writeFile + atomic_copy(localFilePath, absPath) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/lib/misc.py", line 225, in atomic_copy + shutil.copyfile(src_path, dest_path_tmp) + File "/usr/lib/python3.8/shutil.py", line 272, in copyfile + _fastcopy_sendfile(fsrc, fdst) + File "/usr/lib/python3.8/shutil.py", line 163, in _fastcopy_sendfile + raise err from None + File "/usr/lib/python3.8/shutil.py", line 149, in _fastcopy_sendfile + sent = os.sendfile(outfd, infd, offset, blocksize) + OSError: [Errno 28] No space left on device: '/home/drkthomp/gerald_D1VCPACXX_1_1.fastq' -> '/home/drkthomp/protect3/jobStore/files/for-job/kind-prepare_samples/instance-ut9mbtly/file-1az1qk1s/gerald_D1VCPACXX_1_1.fastq.a1093129-8bb8-4954-b599-c68c523a5b6d.tmp.fastq' + [2020-10-31T04:42:48-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-31T04:42:49-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'prepare_samples' kind-prepare_samples/instance-ut9mbtly with ID kind-prepare_samples/instance-ut9mbtly to 0 +[2020-10-31T04:42:49-0700] [MainThread] [W] [toil.leader] Job 'prepare_samples' kind-prepare_samples/instance-ut9mbtly with ID kind-prepare_samples/instance-ut9mbtly is completely failed +[2020-10-31T04:42:57-0700] [MainThread] [I] [toil.leader] Finished toil run with 3 failed jobs. +[2020-10-31T04:42:57-0700] [MainThread] [I] [toil.leader] Failed jobs at end of the run: 'get_all_tool_inputs' kind-parse_config_file/instance-s_nr25h6 'prepare_samples' kind-prepare_samples/instance-ut9mbtly 'launch_protect' kind-launch_protect/instance-_8136hgg +Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/bin/ProTECT", line 33, in + sys.exit(load_entry_point('protect', 'console_scripts', 'ProTECT')()) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1025, in main + Job.Runner.startToil(start, params) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 798, in startToil + return toil.restart() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/common.py", line 878, in restart + return self._runMainLoop(rootJobGraph) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/common.py", line 1133, in _runMainLoop + return Leader(config=self.config, + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/leader.py", line 269, in run + raise FailedJobsException(self.config.jobStore, self.toilState.totalFailedJobs, self.jobStore) +toil.leader.FailedJobsException: The job store 'file:/home/drkthomp/protect3/jobStore' contains 3 failed jobs: 'get_all_tool_inputs' kind-parse_config_file/instance-s_nr25h6, 'prepare_samples' kind-prepare_samples/instance-ut9mbtly, 'launch_protect' kind-launch_protect/instance-_8136hgg +Log from job 'prepare_samples' kind-prepare_samples/instance-ut9mbtly follows: +=========> + [2020-10-31T04:34:44-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-31T04:34:44-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + [2020-10-31T04:34:44-0700] [MainThread] [W] [toil.fileStores.nonCachingFileStore] Starting job files/for-job/kind-launch_protect/instance-_8136hgg/cleanup/file-rbhwf616/stream with less than 10% of disk space remaining. + [2020-10-31T04:34:44-0700] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Downloading Inputs for TEST + [2020-10-31T04:34:44-0700] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (TEST:tumor_dna_fastq_1) to the file job store + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 840, in prepare_samples + output_dict[input_file] = get_pipeline_inputs( + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 805, in get_pipeline_inputs + input_file = job.fileStore.writeGlobalFile(input_file) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/fileStores/nonCachingFileStore.py", line 91, in writeGlobalFile + fileStoreID = self.jobStore.writeFile(absLocalFileName, creatorID, cleanup) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/jobStores/fileJobStore.py", line 387, in writeFile + atomic_copy(localFilePath, absPath) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/lib/misc.py", line 225, in atomic_copy + shutil.copyfile(src_path, dest_path_tmp) + File "/usr/lib/python3.8/shutil.py", line 272, in copyfile + _fastcopy_sendfile(fsrc, fdst) + File "/usr/lib/python3.8/shutil.py", line 163, in _fastcopy_sendfile + raise err from None + File "/usr/lib/python3.8/shutil.py", line 149, in _fastcopy_sendfile + sent = os.sendfile(outfd, infd, offset, blocksize) + OSError: [Errno 28] No space left on device: '/home/drkthomp/gerald_D1VCPACXX_1_1.fastq' -> '/home/drkthomp/protect3/jobStore/files/for-job/kind-prepare_samples/instance-ut9mbtly/file-1az1qk1s/gerald_D1VCPACXX_1_1.fastq.a1093129-8bb8-4954-b599-c68c523a5b6d.tmp.fastq' + [2020-10-31T04:42:48-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= diff --git a/errors/2020-11-08-15-44-22.txt b/errors/2020-11-08-15-44-22.txt new file mode 100644 index 00000000..98ab18e1 --- /dev/null +++ b/errors/2020-11-08-15-44-22.txt @@ -0,0 +1,76 @@ +[2020-11-08T15:44:25-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-get_pipeline_inputs/instance-5vyrv57t +[2020-11-08T15:44:25-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-get_pipeline_inputs/instance-so6a9eh_ +[2020-11-08T15:44:25-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-parse_config_file/instance-buse8sx7 +[2020-11-08T15:44:25-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-08T15:44:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ with job batch system ID: 0 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:44:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-0q41m4jt with job batch system ID: 1 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:44:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-r_s003vc with job batch system ID: 2 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:44:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-zzulcz80 with job batch system ID: 3 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:44:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5vyrv57t with job batch system ID: 4 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:44:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-0x52rixc with job batch system ID: 5 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:44:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-f8kem078 with job batch system ID: 6 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:44:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-10qlus8x with job batch system ID: 7 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:44:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-bqg5g1te with job batch system ID: 8 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:44:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-pvsm6vla with job batch system ID: 9 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:44:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-homdbmzy with job batch system ID: 10 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:44:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmp8uvd3pj7/worker_log.txt +[2020-11-08T15:44:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmp6h7v4bxr/worker_log.txt +[2020-11-08T15:44:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmpruac22uh/worker_log.txt +[2020-11-08T15:44:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmpgozb4up8/worker_log.txt +[2020-11-08T15:44:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmp77x8qqxp/worker_log.txt +[2020-11-08T15:44:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmp15os55up/worker_log.txt +[2020-11-08T15:44:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmp4in9pe3b/worker_log.txt +[2020-11-08T15:44:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmp4j_p2rm4/worker_log.txt +[2020-11-08T15:44:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmp4dcbp4_r/worker_log.txt +[2020-11-08T15:44:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmp43yt9b61/worker_log.txt +[2020-11-08T15:44:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmpr3rsu5l4/worker_log.txt +[2020-11-08T15:44:27-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker get_pipeline_inputs file:/private/home/drkthomp/protect3/jobStore kind-get_pipeline_inputs/instance-so6a9eh_. +[2020-11-08T15:44:27-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ +[2020-11-08T15:44:27-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ +[2020-11-08T15:44:27-0800] [MainThread] [W] [toil.leader] Log from job kind-get_pipeline_inputs/instance-so6a9eh_ follows: +=========> + [2020-11-08T15:44:26-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-08T15:44:26-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-08T15:44:27-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (star:index) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 804, in get_pipeline_inputs + assert os.path.exists(input_file), 'Bogus Input : ' + input_file + AssertionError: Bogus Input : /scratch/drkthomp/protect-indexs/protect-data/star_with_fusion_100bp_readlen_indexes.tar.gz + [2020-11-08T15:44:27-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-08T15:44:27-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ with ID kind-get_pipeline_inputs/instance-so6a9eh_ to 1 +[2020-11-08T15:44:27-0800] [MainThread] [I] [toil.leader] 10 jobs are running, 0 jobs are issued and waiting to run +[2020-11-08T15:44:27-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ with job batch system ID: 11 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:44:28-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmpnwvsq_1c/worker_log.txt +[2020-11-08T15:44:28-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker get_pipeline_inputs file:/private/home/drkthomp/protect3/jobStore kind-get_pipeline_inputs/instance-so6a9eh_. +[2020-11-08T15:44:28-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ +[2020-11-08T15:44:28-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ +[2020-11-08T15:44:28-0800] [MainThread] [W] [toil.leader] Log from job kind-get_pipeline_inputs/instance-so6a9eh_ follows: +=========> + [2020-11-08T15:44:28-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-08T15:44:28-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-08T15:44:28-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (star:index) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 804, in get_pipeline_inputs + assert os.path.exists(input_file), 'Bogus Input : ' + input_file + AssertionError: Bogus Input : /scratch/drkthomp/protect-indexs/protect-data/star_with_fusion_100bp_readlen_indexes.tar.gz + [2020-11-08T15:44:28-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-08T15:44:28-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ with ID kind-get_pipeline_inputs/instance-so6a9eh_ to 0 +[2020-11-08T15:44:29-0800] [MainThread] [W] [toil.leader] Job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ with ID kind-get_pipeline_inputs/instance-so6a9eh_ is completely failed diff --git a/errors/2020-11-08-15-45-24.txt b/errors/2020-11-08-15-45-24.txt new file mode 100644 index 00000000..8cad808e --- /dev/null +++ b/errors/2020-11-08-15-45-24.txt @@ -0,0 +1,78 @@ +[2020-11-08T15:45:25-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-get_pipeline_inputs/instance-so6a9eh_ +[2020-11-08T15:45:25-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-08T15:45:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ with job batch system ID: 0 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:45:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-0q41m4jt with job batch system ID: 1 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:45:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-r_s003vc with job batch system ID: 2 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:45:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-zzulcz80 with job batch system ID: 3 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:45:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5vyrv57t with job batch system ID: 4 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:45:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-0x52rixc with job batch system ID: 5 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:45:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-f8kem078 with job batch system ID: 6 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:45:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-10qlus8x with job batch system ID: 7 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:45:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-bqg5g1te with job batch system ID: 8 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:45:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-pvsm6vla with job batch system ID: 9 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:45:25-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-homdbmzy with job batch system ID: 10 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:45:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmpm1merpt5/worker_log.txt +[2020-11-08T15:45:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmpyan4eo4t/worker_log.txt +[2020-11-08T15:45:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmpp3fm_kwo/worker_log.txt +[2020-11-08T15:45:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmp5phncvve/worker_log.txt +[2020-11-08T15:45:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmptjk0fe3i/worker_log.txt +[2020-11-08T15:45:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmp1xzx7dx0/worker_log.txt +[2020-11-08T15:45:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmpsvhxjx_v/worker_log.txt +[2020-11-08T15:45:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmpajrovz36/worker_log.txt +[2020-11-08T15:45:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmpryfd351h/worker_log.txt +[2020-11-08T15:45:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmphoo674qz/worker_log.txt +[2020-11-08T15:45:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmpsdhw54gz/worker_log.txt +[2020-11-08T15:45:27-0800] [MainThread] [I] [toil.leader] 11 jobs are running, 0 jobs are issued and waiting to run +[2020-11-08T15:45:30-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker get_pipeline_inputs file:/private/home/drkthomp/protect3/jobStore kind-get_pipeline_inputs/instance-so6a9eh_. +[2020-11-08T15:45:30-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ +[2020-11-08T15:45:30-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ +[2020-11-08T15:45:30-0800] [MainThread] [W] [toil.leader] Log from job kind-get_pipeline_inputs/instance-so6a9eh_ follows: +=========> + [2020-11-08T15:45:26-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-08T15:45:26-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-08T15:45:27-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (star:index) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 804, in get_pipeline_inputs + assert os.path.exists(input_file), 'Bogus Input : ' + input_file + AssertionError: Bogus Input : /scratch/drkthomp/protect-indexs/protect-data/star_with_fusion_100bp_readlen_indexes.tar.gz + [2020-11-08T15:45:27-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-08T15:45:30-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ with ID kind-get_pipeline_inputs/instance-so6a9eh_ to 1 +[2020-11-08T15:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ with job batch system ID: 11 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-08T15:45:31-0800] [MainThread] [I] [toil.worker] Redirecting logging to /tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmpsvow2yvr/worker_log.txt +[2020-11-08T15:45:32-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-08-2020 15:45:32: Obtaining file (transgene:gencode_annotation_gtf) to the file job store +[2020-11-08T15:45:33-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker get_pipeline_inputs file:/private/home/drkthomp/protect3/jobStore kind-get_pipeline_inputs/instance-so6a9eh_. +[2020-11-08T15:45:33-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ +[2020-11-08T15:45:33-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ +[2020-11-08T15:45:33-0800] [MainThread] [W] [toil.leader] Log from job kind-get_pipeline_inputs/instance-so6a9eh_ follows: +=========> + [2020-11-08T15:45:31-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-08T15:45:31-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-08T15:45:32-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (star:index) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 804, in get_pipeline_inputs + assert os.path.exists(input_file), 'Bogus Input : ' + input_file + AssertionError: Bogus Input : /scratch/drkthomp/protect-indexs/protect-data/star_with_fusion_100bp_readlen_indexes.tar.gz + [2020-11-08T15:45:32-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-08T15:45:33-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ with ID kind-get_pipeline_inputs/instance-so6a9eh_ to 0 +[2020-11-08T15:45:33-0800] [MainThread] [W] [toil.leader] Job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ with ID kind-get_pipeline_inputs/instance-so6a9eh_ is completely failed +[2020-11-08T15:45:33-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-bqg5g1te +[2020-11-08T15:45:34-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-08-2020 15:45:34: Obtaining file (indexes:cosmic_vcf) to the file job store +[2020-11-08T15:45:34-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5vyrv57t diff --git a/errors/2020-11-10-17-39-03.txt b/errors/2020-11-10-17-39-03.txt new file mode 100644 index 00000000..f853f43c --- /dev/null +++ b/errors/2020-11-10-17-39-03.txt @@ -0,0 +1,43 @@ +[2020-11-10T17:39:06-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-get_pipeline_inputs/instance-so6a9eh_ +[2020-11-10T17:39:06-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-parse_config_file/instance-buse8sx7 +[2020-11-10T17:39:06-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-10T17:39:06-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ with job batch system ID: 0 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:39:06-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-0q41m4jt with job batch system ID: 1 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:39:06-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-r_s003vc with job batch system ID: 2 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:39:06-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-zzulcz80 with job batch system ID: 3 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:39:06-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-0x52rixc with job batch system ID: 4 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:39:06-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-f8kem078 with job batch system ID: 5 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:39:06-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-10qlus8x with job batch system ID: 6 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:39:06-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-pvsm6vla with job batch system ID: 7 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:39:06-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-homdbmzy with job batch system ID: 8 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:39:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmpb8x0hq8i/worker_log.txt +[2020-11-10T17:39:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmp6hq_z8g7/worker_log.txt +[2020-11-10T17:39:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmp1r52fuck/worker_log.txt +[2020-11-10T17:39:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmpwovno3lt/worker_log.txt +[2020-11-10T17:39:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmpfwaduu58/worker_log.txt +[2020-11-10T17:39:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmp2zwut9oe/worker_log.txt +[2020-11-10T17:39:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmpq1nj79iy/worker_log.txt +[2020-11-10T17:39:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmp9uuvkco7/worker_log.txt +[2020-11-10T17:39:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-b3157790-d4c6-49ae-8620-5f2a1485ceb4-e8611b53-bdab-4598-acae-87977f99831e/tmp0p26_5hg/worker_log.txt +[2020-11-10T17:39:08-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker get_pipeline_inputs file:/private/home/drkthomp/protect3/jobStore kind-get_pipeline_inputs/instance-so6a9eh_. +[2020-11-10T17:39:08-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ +[2020-11-10T17:39:08-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-so6a9eh_ +[2020-11-10T17:39:08-0800] [MainThread] [W] [toil.leader] Log from job kind-get_pipeline_inputs/instance-so6a9eh_ follows: +=========> + [2020-11-10T17:39:07-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-10T17:39:07-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-10T17:39:07-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (star:index) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 804, in get_pipeline_inputs + assert os.path.exists(input_file), 'Bogus Input : ' + input_file + AssertionError: Bogus Input : /scratch/drkthomp/protect-indexs/protect-data/star_with_fusion_100bp_readlen_indexes.tar.gz + [2020-11-10T17:39:07-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= diff --git a/errors/2020-11-10-17-47-22.txt b/errors/2020-11-10-17-47-22.txt new file mode 100644 index 00000000..b25fd5e2 --- /dev/null +++ b/errors/2020-11-10-17-47-22.txt @@ -0,0 +1,25 @@ +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 556, in readSharedFileStream + with open(self._getSharedFilePath(sharedFileName), 'rb') as f: +FileNotFoundError: [Errno 2] No such file or directory: '/private/home/drkthomp/protect3/jobStore/files/shared/config.pickle' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/bin/ProTECT", line 11, in + load_entry_point('protect', 'console_scripts', 'ProTECT')() + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1025, in main + Job.Runner.startToil(start, params) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 794, in startToil + with Toil(options) as toil: + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 776, in __enter__ + jobStore.resume() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 122, in resume + super(FileJobStore, self).resume() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/abstractJobStore.py", line 164, in resume + with self.readSharedFileStream('config.pickle') as fileHandle: + File "/usr/lib64/python3.6/contextlib.py", line 81, in __enter__ + return next(self.gen) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 560, in readSharedFileStream + raise NoSuchFileException(sharedFileName) +toil.jobStores.abstractJobStore.NoSuchFileException: File 'config.pickle' does not exist. diff --git a/errors/2020-11-10-17-47-50.txt b/errors/2020-11-10-17-47-50.txt new file mode 100644 index 00000000..6c274c86 --- /dev/null +++ b/errors/2020-11-10-17-47-50.txt @@ -0,0 +1,46 @@ +usage: ProTECT [-h] (--config_file CONFIG_FILE | --generate_config) + [--max-cores-per-job MAX_CORES] [--logOff] [--logCritical] + [--logError] [--logWarning] [--logInfo] [--logDebug] + [--logLevel LOGLEVEL] [--logFile LOGFILE] [--rotatingLogging] + [--workDir WORKDIR] [--noStdOutErr] [--stats] + [--clean {always,onError,never,onSuccess}] + [--cleanWorkDir {always,never,onSuccess,onError}] + [--clusterStats [CLUSTERSTATS]] [--restart] + [--batchSystem BATCHSYSTEM] [--disableHotDeployment] + [--disableAutoDeployment] [--maxLocalJobs MAXLOCALJOBS] + [--manualMemArgs] [--runCwlInternalJobsOnWorkers] + [--parasolCommand PARASOLCOMMAND] + [--parasolMaxBatches PARASOLMAXBATCHES] [--scale SCALE] + [--linkImports] [--moveExports] + [--mesosMaster MESOSMASTERADDRESS] + [--kubernetesHostPath KUBERNETESHOSTPATH] + [--provisioner {aws,gce}] [--nodeTypes NODETYPES] + [--nodeOptions NODEOPTIONS] [--minNodes MINNODES] + [--maxNodes MAXNODES] [--targetTime TARGETTIME] + [--betaInertia BETAINERTIA] [--scaleInterval SCALEINTERVAL] + [--preemptableCompensation PREEMPTABLECOMPENSATION] + [--nodeStorage NODESTORAGE] + [--nodeStorageOverrides NODESTORAGEOVERRIDES] [--metrics] + [--maxServiceJobs MAXSERVICEJOBS] + [--maxPreemptableServiceJobs MAXPREEMPTABLESERVICEJOBS] + [--deadlockWait DEADLOCKWAIT] + [--deadlockCheckInterval DEADLOCKCHECKINTERVAL] + [--statePollingWait STATEPOLLINGWAIT] [--defaultMemory INT] + [--defaultCores FLOAT] [--defaultDisk INT] + [--defaultPreemptable] [--maxCores INT] [--maxMemory INT] + [--maxDisk INT] [--retryCount RETRYCOUNT] + [--enableUnlimitedPreemptableRetries] + [--maxJobDuration MAXJOBDURATION] + [--rescueJobsFrequency RESCUEJOBSFREQUENCY] + [--disableCaching [DISABLECACHING]] [--disableChaining] + [--disableJobStoreChecksumVerification] + [--maxLogFileSize MAXLOGFILESIZE] [--writeLogs [WRITELOGS]] + [--writeLogsGzip [WRITELOGSGZIP]] [--writeLogsFromAllJobs] + [--realTimeLogging] [--sseKey SSEKEY] + [--setEnv NAME=VALUE or NAME] + [--servicePollingInterval SERVICEPOLLINGINTERVAL] + [--forceDockerAppliance] [--disableProgress] [--debugWorker] + [--disableWorkerOutputCapture] [--badWorker BADWORKER] + [--badWorkerFailInterval BADWORKERFAILINTERVAL] + jobStore +ProTECT: error: argument --clean: expected one argument diff --git a/errors/2020-11-10-17-48-39.txt b/errors/2020-11-10-17-48-39.txt new file mode 100644 index 00000000..8ad0eae4 --- /dev/null +++ b/errors/2020-11-10-17-48-39.txt @@ -0,0 +1,19 @@ +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 104, in initialize + os.mkdir(self.jobStoreDir) +FileExistsError: [Errno 17] File exists: '/private/home/drkthomp/protect3/jobStore' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/bin/ProTECT", line 11, in + load_entry_point('protect', 'console_scripts', 'ProTECT')() + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1025, in main + Job.Runner.startToil(start, params) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 794, in startToil + with Toil(options) as toil: + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 774, in __enter__ + jobStore.initialize(config) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 107, in initialize + raise JobStoreExistsException(self.jobStoreDir) +toil.jobStores.abstractJobStore.JobStoreExistsException: The job store '/private/home/drkthomp/protect3/jobStore' already exists. Use --restart to resume the workflow, or remove the job store with 'toil clean' to start the workflow from scratch. diff --git a/errors/2020-11-10-17-48-52.txt b/errors/2020-11-10-17-48-52.txt new file mode 100644 index 00000000..b5602b39 --- /dev/null +++ b/errors/2020-11-10-17-48-52.txt @@ -0,0 +1,263 @@ +[2020-11-10T17:48:54-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-10T17:48:54-0800] [MainThread] [I] [toil.leader] Issued job 'parse_config_file' kind-parse_config_file/instance-x3p_8ykw with job batch system ID: 0 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:48:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpbq02ictx/worker_log.txt +[2020-11-10T17:48:56-0800] [MainThread] [I] [toil.leader] 1 jobs are running, 0 jobs are issued and waiting to run +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Job ended: 'parse_config_file' kind-parse_config_file/instance-x3p_8ykw +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy with job batch system ID: 1 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-v_8wzsqv with job batch system ID: 2 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1suh4rn4 with job batch system ID: 3 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-kygodbz9 with job batch system ID: 4 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-zc74xon7 with job batch system ID: 5 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-cco2yxpr with job batch system ID: 6 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-bqx3_o7f with job batch system ID: 7 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-trjsz99m with job batch system ID: 8 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-v5t1yfbr with job batch system ID: 9 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-vpdl5khq with job batch system ID: 10 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-8f4u6rj6 with job batch system ID: 11 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-2tzj19an with job batch system ID: 12 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-h1ie9fz0 with job batch system ID: 13 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-zcepgoz4 with job batch system ID: 14 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-bj5qi28z with job batch system ID: 15 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-hdqqeskf with job batch system ID: 16 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-e96y5wi7 with job batch system ID: 17 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ab38bwje with job batch system ID: 18 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-qgbish4p with job batch system ID: 19 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-jtthf2_z with job batch system ID: 20 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-nvbw6s1o with job batch system ID: 21 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ix62e3al with job batch system ID: 22 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ipek7skr with job batch system ID: 23 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-cawqgply with job batch system ID: 24 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1nfvnwry with job batch system ID: 25 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-8qpv_clw with job batch system ID: 26 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-0ryf7s78 with job batch system ID: 27 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-_6hqw3qp with job batch system ID: 28 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-zrjfnl8x with job batch system ID: 29 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T17:49:04-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 17:49:04: Parsing config file +[2020-11-10T17:49:04-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 17:49:04: Obtaining tool inputs +[2020-11-10T17:49:04-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 17:49:04: Obtained tool inputs +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpa4w2xk40/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpauucdc3h/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpij9whc8q/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpoydp9vy5/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpyozjemjm/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmp8v__12r8/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpwm0v70j5/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmp9dzv3zw6/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmp92bdkmvu/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmp0pmj43ro/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmp4t9n4f4_/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmp_4znejyb/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmp85da_vc2/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpog1yy_tx/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpnkto5wp_/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpydrbf4mj/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpw1kwv2jh/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpss8vf676/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpqrl_gupt/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpwkbf794h/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpr3wibnr3/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmp1zjnm7xd/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmp9iw0awj7/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmprzfezyjb/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmppn0dj2vt/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmp8s42hcpp/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmphn74gp2m/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpjccc5obv/worker_log.txt +[2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmp_7iulkp4/worker_log.txt +[2020-11-10T17:49:08-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-zc74xon7 +[2020-11-10T17:50:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 17:50:48: Obtaining file (indexes:genome_fai) to the file job store +[2020-11-10T18:07:13-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-trjsz99m +[2020-11-10T18:13:41-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-vpdl5khq +[2020-11-10T18:13:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:42: Obtaining file (indexes:cosmic_idx) to the file job store +[2020-11-10T18:13:42-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-8f4u6rj6 +[2020-11-10T18:13:43-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker get_pipeline_inputs file:/private/home/drkthomp/protect3/jobStore kind-get_pipeline_inputs/instance-jtthf2_z. +[2020-11-10T18:13:43-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-jtthf2_z +[2020-11-10T18:13:43-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-jtthf2_z +[2020-11-10T18:13:43-0800] [MainThread] [W] [toil.leader] Log from job kind-get_pipeline_inputs/instance-jtthf2_z follows: +=========> + [2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-10T17:49:06-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-10T17:49:07-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (transgene:gencode_transcript_fasta) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 805, in get_pipeline_inputs + input_file = job.fileStore.writeGlobalFile(input_file) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/fileStores/nonCachingFileStore.py", line 91, in writeGlobalFile + fileStoreID = self.jobStore.writeFile(absLocalFileName, creatorID, cleanup) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 387, in writeFile + atomic_copy(localFilePath, absPath) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/lib/misc.py", line 225, in atomic_copy + shutil.copyfile(src_path, dest_path_tmp) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 122, in copyfile + copyfileobj(fsrc, fdst) + OSError: [Errno 122] Disk quota exceeded + [2020-11-10T18:13:41-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-10T18:13:47-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:47: Obtaining file (indexes:dbsnp_idx) to the file job store +[2020-11-10T18:13:47-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:47: Obtaining file (indexes:dbsnp_tbi) to the file job store +[2020-11-10T18:13:47-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:47: Obtaining file (radia:retrogene_beds) to the file job store +[2020-11-10T18:13:47-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:47: Obtaining file (indexes:genome_dict) to the file job store +[2020-11-10T18:13:47-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:47: Obtaining file (radia:gencode_beds) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (transgene:gencode_annotation_gtf) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (radia:dbsnp_beds) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (transgene:gencode_peptide_fasta) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (strelka:config_file) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (indexes:cosmic_vcf) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (radia:pseudogene_beds) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (snpeff:index) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (reports:itx_resistance_file) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (rsem:index) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (mhci:method_file) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (reports:immune_resistance_pathways_file) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (radia:cosmic_beds) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (indexes:genome_fasta) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (phlat:index) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (indexes:dbsnp_vcf) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (reports:mhc_pathways_file) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (reports:car_t_targets_file) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (bwa:index) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (mhcii:method_file) to the file job store +[2020-11-10T18:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:13:48: Obtaining file (transgene:genome_fasta) to the file job store +[2020-11-10T18:13:54-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker get_pipeline_inputs file:/private/home/drkthomp/protect3/jobStore kind-get_pipeline_inputs/instance-6sm7y6xy. +[2020-11-10T18:14:01-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-jtthf2_z with ID kind-get_pipeline_inputs/instance-jtthf2_z to 1 +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-jtthf2_z with job batch system ID: 30 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-zcepgoz4 +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-cco2yxpr +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-hdqqeskf +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-nvbw6s1o +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-h1ie9fz0 +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-e96y5wi7 +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-qgbish4p +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-bqx3_o7f +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-bj5qi28z +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-0ryf7s78 +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-cawqgply +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ab38bwje +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1suh4rn4 +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-2tzj19an +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-_6hqw3qp +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-kygodbz9 +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ipek7skr +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-v5t1yfbr +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-8qpv_clw +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-zrjfnl8x +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-v_8wzsqv +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1nfvnwry +[2020-11-10T18:14:01-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ix62e3al +[2020-11-10T18:14:01-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy +[2020-11-10T18:14:01-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy +[2020-11-10T18:14:01-0800] [MainThread] [W] [toil.leader] Log from job kind-get_pipeline_inputs/instance-6sm7y6xy follows: +=========> + [2020-11-10T17:49:06-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-10T17:49:06-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-10T17:49:06-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (star:index) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 805, in get_pipeline_inputs + input_file = job.fileStore.writeGlobalFile(input_file) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/fileStores/nonCachingFileStore.py", line 91, in writeGlobalFile + fileStoreID = self.jobStore.writeFile(absLocalFileName, creatorID, cleanup) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 387, in writeFile + atomic_copy(localFilePath, absPath) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/lib/misc.py", line 225, in atomic_copy + shutil.copyfile(src_path, dest_path_tmp) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 122, in copyfile + copyfileobj(fsrc, fdst) + OSError: [Errno 122] Disk quota exceeded + [2020-11-10T18:13:54-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-10T18:14:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpo1mr33dn/worker_log.txt +[2020-11-10T18:14:04-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-10-2020 18:14:04: Obtaining file (transgene:gencode_transcript_fasta) to the file job store +[2020-11-10T18:14:11-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy with ID kind-get_pipeline_inputs/instance-6sm7y6xy to 1 +[2020-11-10T18:14:11-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy with job batch system ID: 31 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-10T18:14:11-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-jtthf2_z +[2020-11-10T18:14:12-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmp1ynm_k6z/worker_log.txt +[2020-11-10T18:28:55-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker get_pipeline_inputs file:/private/home/drkthomp/protect3/jobStore kind-get_pipeline_inputs/instance-6sm7y6xy. +[2020-11-10T18:28:55-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy +[2020-11-10T18:28:55-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy +[2020-11-10T18:28:55-0800] [MainThread] [W] [toil.leader] Log from job kind-get_pipeline_inputs/instance-6sm7y6xy follows: +=========> + [2020-11-10T18:14:12-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-10T18:14:12-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-10T18:14:12-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (star:index) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 805, in get_pipeline_inputs + input_file = job.fileStore.writeGlobalFile(input_file) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/fileStores/nonCachingFileStore.py", line 91, in writeGlobalFile + fileStoreID = self.jobStore.writeFile(absLocalFileName, creatorID, cleanup) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 387, in writeFile + atomic_copy(localFilePath, absPath) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/lib/misc.py", line 225, in atomic_copy + shutil.copyfile(src_path, dest_path_tmp) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 122, in copyfile + copyfileobj(fsrc, fdst) + OSError: [Errno 122] Disk quota exceeded + [2020-11-10T18:28:55-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-10T18:29:05-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy with ID kind-get_pipeline_inputs/instance-6sm7y6xy to 0 +[2020-11-10T18:29:05-0800] [MainThread] [W] [toil.leader] Job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy with ID kind-get_pipeline_inputs/instance-6sm7y6xy is completely failed +[2020-11-10T18:29:12-0800] [MainThread] [I] [toil.leader] Finished toil run with 2 failed jobs. +[2020-11-10T18:29:12-0800] [MainThread] [I] [toil.leader] Failed jobs at end of the run: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy 'get_all_tool_inputs' kind-parse_config_file/instance-x3p_8ykw +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/bin/ProTECT", line 11, in + load_entry_point('protect', 'console_scripts', 'ProTECT')() + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1025, in main + Job.Runner.startToil(start, params) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 796, in startToil + return toil.start(job) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 846, in start + return self._runMainLoop(rootJobGraph) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 1138, in _runMainLoop + jobCache=self._jobCache).run() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/leader.py", line 269, in run + raise FailedJobsException(self.config.jobStore, self.toilState.totalFailedJobs, self.jobStore) +toil.leader.FailedJobsException: The job store 'file:/private/home/drkthomp/protect3/jobStore' contains 2 failed jobs: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy, 'get_all_tool_inputs' kind-parse_config_file/instance-x3p_8ykw +Log from job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy follows: +=========> + [2020-11-10T18:14:12-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-10T18:14:12-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-10T18:14:12-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (star:index) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 805, in get_pipeline_inputs + input_file = job.fileStore.writeGlobalFile(input_file) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/fileStores/nonCachingFileStore.py", line 91, in writeGlobalFile + fileStoreID = self.jobStore.writeFile(absLocalFileName, creatorID, cleanup) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 387, in writeFile + atomic_copy(localFilePath, absPath) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/lib/misc.py", line 225, in atomic_copy + shutil.copyfile(src_path, dest_path_tmp) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 122, in copyfile + copyfileobj(fsrc, fdst) + OSError: [Errno 122] Disk quota exceeded + [2020-11-10T18:28:55-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= diff --git a/errors/2020-11-11-13-41-03.txt b/errors/2020-11-11-13-41-03.txt new file mode 100644 index 00000000..6a33dc5b --- /dev/null +++ b/errors/2020-11-11-13-41-03.txt @@ -0,0 +1,114 @@ +[2020-11-11T13:41:05-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-parse_config_file/instance-x3p_8ykw +[2020-11-11T13:41:05-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-get_pipeline_inputs/instance-6sm7y6xy +[2020-11-11T13:41:05-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-11T13:41:05-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy with job batch system ID: 0 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-11T13:41:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpc802qz61/worker_log.txt +[2020-11-11T13:41:07-0800] [MainThread] [I] [toil.leader] 1 jobs are running, 0 jobs are issued and waiting to run +[2020-11-11T13:53:42-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker get_pipeline_inputs file:/private/home/drkthomp/protect3/jobStore kind-get_pipeline_inputs/instance-6sm7y6xy. +[2020-11-11T13:53:42-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy +[2020-11-11T13:53:42-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy +[2020-11-11T13:53:42-0800] [MainThread] [W] [toil.leader] Log from job kind-get_pipeline_inputs/instance-6sm7y6xy follows: +=========> + [2020-11-11T13:41:06-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-11T13:41:06-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-11T13:41:06-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (star:index) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 805, in get_pipeline_inputs + input_file = job.fileStore.writeGlobalFile(input_file) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/fileStores/nonCachingFileStore.py", line 91, in writeGlobalFile + fileStoreID = self.jobStore.writeFile(absLocalFileName, creatorID, cleanup) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 387, in writeFile + atomic_copy(localFilePath, absPath) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/lib/misc.py", line 225, in atomic_copy + shutil.copyfile(src_path, dest_path_tmp) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 122, in copyfile + copyfileobj(fsrc, fdst) + OSError: [Errno 122] Disk quota exceeded + [2020-11-11T13:53:42-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-11T13:53:53-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy with ID kind-get_pipeline_inputs/instance-6sm7y6xy to 1 +[2020-11-11T13:53:53-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy with job batch system ID: 1 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-11T13:53:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/tmp/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpuxo7h5ih/worker_log.txt +[2020-11-11T14:05:52-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker get_pipeline_inputs file:/private/home/drkthomp/protect3/jobStore kind-get_pipeline_inputs/instance-6sm7y6xy. +[2020-11-11T14:05:52-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy +[2020-11-11T14:05:52-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy +[2020-11-11T14:05:52-0800] [MainThread] [W] [toil.leader] Log from job kind-get_pipeline_inputs/instance-6sm7y6xy follows: +=========> + [2020-11-11T13:53:54-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-11T13:53:54-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-11T13:53:56-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (star:index) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 805, in get_pipeline_inputs + input_file = job.fileStore.writeGlobalFile(input_file) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/fileStores/nonCachingFileStore.py", line 91, in writeGlobalFile + fileStoreID = self.jobStore.writeFile(absLocalFileName, creatorID, cleanup) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 387, in writeFile + atomic_copy(localFilePath, absPath) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/lib/misc.py", line 225, in atomic_copy + shutil.copyfile(src_path, dest_path_tmp) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 122, in copyfile + copyfileobj(fsrc, fdst) + OSError: [Errno 122] Disk quota exceeded + [2020-11-11T14:05:51-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-11T14:06:02-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy with ID kind-get_pipeline_inputs/instance-6sm7y6xy to 0 +[2020-11-11T14:06:02-0800] [MainThread] [W] [toil.leader] Job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy with ID kind-get_pipeline_inputs/instance-6sm7y6xy is completely failed +[2020-11-11T14:06:08-0800] [MainThread] [I] [toil.leader] Finished toil run with 2 failed jobs. +[2020-11-11T14:06:08-0800] [MainThread] [I] [toil.leader] Failed jobs at end of the run: 'get_all_tool_inputs' kind-parse_config_file/instance-x3p_8ykw 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/bin/ProTECT", line 11, in + load_entry_point('protect', 'console_scripts', 'ProTECT')() + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1025, in main + Job.Runner.startToil(start, params) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 798, in startToil + return toil.restart() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 878, in restart + return self._runMainLoop(rootJobGraph) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 1138, in _runMainLoop + jobCache=self._jobCache).run() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/leader.py", line 269, in run + raise FailedJobsException(self.config.jobStore, self.toilState.totalFailedJobs, self.jobStore) +toil.leader.FailedJobsException: The job store 'file:/private/home/drkthomp/protect3/jobStore' contains 2 failed jobs: 'get_all_tool_inputs' kind-parse_config_file/instance-x3p_8ykw, 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy +Log from job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy follows: +=========> + [2020-11-11T13:53:54-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-11T13:53:54-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-11T13:53:56-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (star:index) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 805, in get_pipeline_inputs + input_file = job.fileStore.writeGlobalFile(input_file) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/fileStores/nonCachingFileStore.py", line 91, in writeGlobalFile + fileStoreID = self.jobStore.writeFile(absLocalFileName, creatorID, cleanup) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 387, in writeFile + atomic_copy(localFilePath, absPath) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/lib/misc.py", line 225, in atomic_copy + shutil.copyfile(src_path, dest_path_tmp) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 122, in copyfile + copyfileobj(fsrc, fdst) + OSError: [Errno 122] Disk quota exceeded + [2020-11-11T14:05:51-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= diff --git a/errors/2020-11-11-16-09-15.txt b/errors/2020-11-11-16-09-15.txt new file mode 100644 index 00000000..daf3bb4a --- /dev/null +++ b/errors/2020-11-11-16-09-15.txt @@ -0,0 +1,113 @@ +[2020-11-11T16:09:17-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-get_pipeline_inputs/instance-6sm7y6xy +[2020-11-11T16:09:17-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-11T16:09:17-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy with job batch system ID: 0 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-11T16:09:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmpzyc5xvot/worker_log.txt +[2020-11-11T16:09:19-0800] [MainThread] [I] [toil.leader] 1 jobs are running, 0 jobs are issued and waiting to run +[2020-11-11T16:22:09-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker get_pipeline_inputs file:/private/home/drkthomp/protect3/jobStore kind-get_pipeline_inputs/instance-6sm7y6xy. +[2020-11-11T16:22:09-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy +[2020-11-11T16:22:09-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy +[2020-11-11T16:22:09-0800] [MainThread] [W] [toil.leader] Log from job kind-get_pipeline_inputs/instance-6sm7y6xy follows: +=========> + [2020-11-11T16:09:18-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-11T16:09:18-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-11T16:09:19-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (star:index) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 805, in get_pipeline_inputs + input_file = job.fileStore.writeGlobalFile(input_file) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/fileStores/nonCachingFileStore.py", line 91, in writeGlobalFile + fileStoreID = self.jobStore.writeFile(absLocalFileName, creatorID, cleanup) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 387, in writeFile + atomic_copy(localFilePath, absPath) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/lib/misc.py", line 225, in atomic_copy + shutil.copyfile(src_path, dest_path_tmp) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 122, in copyfile + copyfileobj(fsrc, fdst) + OSError: [Errno 122] Disk quota exceeded + [2020-11-11T16:22:08-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-11T16:22:09-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy with ID kind-get_pipeline_inputs/instance-6sm7y6xy to 1 +[2020-11-11T16:22:09-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy with job batch system ID: 1 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-11T16:22:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-40d45df0-0414-4c8c-bfa2-203d7b77e976-e8611b53-bdab-4598-acae-87977f99831e/tmp5x08rufj/worker_log.txt +[2020-11-11T16:36:53-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker get_pipeline_inputs file:/private/home/drkthomp/protect3/jobStore kind-get_pipeline_inputs/instance-6sm7y6xy. +[2020-11-11T16:36:53-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy +[2020-11-11T16:36:53-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy +[2020-11-11T16:36:53-0800] [MainThread] [W] [toil.leader] Log from job kind-get_pipeline_inputs/instance-6sm7y6xy follows: +=========> + [2020-11-11T16:22:09-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-11T16:22:09-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-11T16:22:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (star:index) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 805, in get_pipeline_inputs + input_file = job.fileStore.writeGlobalFile(input_file) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/fileStores/nonCachingFileStore.py", line 91, in writeGlobalFile + fileStoreID = self.jobStore.writeFile(absLocalFileName, creatorID, cleanup) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 387, in writeFile + atomic_copy(localFilePath, absPath) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/lib/misc.py", line 225, in atomic_copy + shutil.copyfile(src_path, dest_path_tmp) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 122, in copyfile + copyfileobj(fsrc, fdst) + OSError: [Errno 122] Disk quota exceeded + [2020-11-11T16:36:53-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-11T16:36:53-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy with ID kind-get_pipeline_inputs/instance-6sm7y6xy to 0 +[2020-11-11T16:36:54-0800] [MainThread] [W] [toil.leader] Job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy with ID kind-get_pipeline_inputs/instance-6sm7y6xy is completely failed +[2020-11-11T16:36:59-0800] [MainThread] [I] [toil.leader] Finished toil run with 2 failed jobs. +[2020-11-11T16:36:59-0800] [MainThread] [I] [toil.leader] Failed jobs at end of the run: 'get_all_tool_inputs' kind-parse_config_file/instance-x3p_8ykw 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/bin/ProTECT", line 11, in + load_entry_point('protect', 'console_scripts', 'ProTECT')() + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1025, in main + Job.Runner.startToil(start, params) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 798, in startToil + return toil.restart() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 878, in restart + return self._runMainLoop(rootJobGraph) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 1138, in _runMainLoop + jobCache=self._jobCache).run() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/leader.py", line 269, in run + raise FailedJobsException(self.config.jobStore, self.toilState.totalFailedJobs, self.jobStore) +toil.leader.FailedJobsException: The job store 'file:/private/home/drkthomp/protect3/jobStore' contains 2 failed jobs: 'get_all_tool_inputs' kind-parse_config_file/instance-x3p_8ykw, 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy +Log from job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6sm7y6xy follows: +=========> + [2020-11-11T16:22:09-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-11T16:22:09-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-11T16:22:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (star:index) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 805, in get_pipeline_inputs + input_file = job.fileStore.writeGlobalFile(input_file) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/fileStores/nonCachingFileStore.py", line 91, in writeGlobalFile + fileStoreID = self.jobStore.writeFile(absLocalFileName, creatorID, cleanup) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 387, in writeFile + atomic_copy(localFilePath, absPath) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/lib/misc.py", line 225, in atomic_copy + shutil.copyfile(src_path, dest_path_tmp) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 122, in copyfile + copyfileobj(fsrc, fdst) + OSError: [Errno 122] Disk quota exceeded + [2020-11-11T16:36:53-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= diff --git a/errors/2020-11-13-15-32-47.txt b/errors/2020-11-13-15-32-47.txt new file mode 100644 index 00000000..b43dd0bf --- /dev/null +++ b/errors/2020-11-13-15-32-47.txt @@ -0,0 +1,12 @@ +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/bin/ProTECT", line 11, in + load_entry_point('protect', 'console_scripts', 'ProTECT')() + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1025, in main + Job.Runner.startToil(start, params) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 794, in startToil + with Toil(options) as toil: + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 776, in __enter__ + jobStore.resume() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 121, in resume + raise NoSuchJobStoreException(self.jobStoreDir) +toil.jobStores.abstractJobStore.NoSuchJobStoreException: The job store '/scratch/drkthomp/jobStore' does not exist, so there is nothing to restart. diff --git a/errors/2020-11-13-15-33-17.txt b/errors/2020-11-13-15-33-17.txt new file mode 100644 index 00000000..ca13dc5a --- /dev/null +++ b/errors/2020-11-13-15-33-17.txt @@ -0,0 +1,222 @@ +[2020-11-13T15:33:20-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-13T15:33:20-0800] [MainThread] [I] [toil.leader] Issued job 'parse_config_file' kind-parse_config_file/instance-qt4s63a_ with job batch system ID: 0 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmp1pn0pbbe/worker_log.txt +[2020-11-13T15:33:22-0800] [MainThread] [I] [toil.leader] 1 jobs are running, 0 jobs are issued and waiting to run +[2020-11-13T15:33:23-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:23: Parsing config file +[2020-11-13T15:33:23-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:23: Obtaining tool inputs +[2020-11-13T15:33:23-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:23: Obtained tool inputs +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Job ended: 'parse_config_file' kind-parse_config_file/instance-qt4s63a_ +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-jq4g_3yc with job batch system ID: 1 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-zx6h9z5d with job batch system ID: 2 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-j6ge7umm with job batch system ID: 3 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-01h704qy with job batch system ID: 4 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1jrfy_07 with job batch system ID: 5 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-09ig6r59 with job batch system ID: 6 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-qac4y69v with job batch system ID: 7 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-7temn04b with job batch system ID: 8 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-tgrluxoj with job batch system ID: 9 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-a6ie8zcq with job batch system ID: 10 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5bdy2vh2 with job batch system ID: 11 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-4io4i1l8 with job batch system ID: 12 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-2i61_7li with job batch system ID: 13 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-mi83gk2v with job batch system ID: 14 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-wf_msw97 with job batch system ID: 15 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5_jfkiu5 with job batch system ID: 16 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-4etp2gh5 with job batch system ID: 17 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-y1gvjtt_ with job batch system ID: 18 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-9dkdlfhu with job batch system ID: 19 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-nur3yi6b with job batch system ID: 20 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-dqxfjdxj with job batch system ID: 21 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-diuz59sb with job batch system ID: 22 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5_apbcx6 with job batch system ID: 23 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-o1f0sosm with job batch system ID: 24 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-4dyd2xg5 with job batch system ID: 25 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-65blz2rt with job batch system ID: 26 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-40mnvizw with job batch system ID: 27 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-eg0mroe5 with job batch system ID: 28 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:23-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-a481ugll with job batch system ID: 29 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:33:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmpnwaz66ju/worker_log.txt +[2020-11-13T15:33:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmpxngkppe4/worker_log.txt +[2020-11-13T15:33:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmph8nxhes7/worker_log.txt +[2020-11-13T15:33:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmp_ll02drp/worker_log.txt +[2020-11-13T15:33:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmpt5mespkq/worker_log.txt +[2020-11-13T15:33:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmppep4rmr6/worker_log.txt +[2020-11-13T15:33:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmp8ve4gf4m/worker_log.txt +[2020-11-13T15:33:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmpwowbh2vk/worker_log.txt +[2020-11-13T15:33:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmpq1dufc3l/worker_log.txt +[2020-11-13T15:33:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmpw8eame5a/worker_log.txt +[2020-11-13T15:33:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmp1_u_m0j8/worker_log.txt +[2020-11-13T15:33:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmp3r6l4mvl/worker_log.txt +[2020-11-13T15:33:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmp9_17iqgu/worker_log.txt +[2020-11-13T15:33:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmp49l0dbd9/worker_log.txt +[2020-11-13T15:33:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmpl11pnj_l/worker_log.txt +[2020-11-13T15:33:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmpayt640ua/worker_log.txt +[2020-11-13T15:33:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmp6dvn0vos/worker_log.txt +[2020-11-13T15:33:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmptu6mbdhg/worker_log.txt +[2020-11-13T15:33:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmpi_3uv720/worker_log.txt +[2020-11-13T15:33:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmpq9w3k8w4/worker_log.txt +[2020-11-13T15:33:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmp2z9lkedu/worker_log.txt +[2020-11-13T15:33:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmpm0oy4_st/worker_log.txt +[2020-11-13T15:33:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmpzy0ra6nb/worker_log.txt +[2020-11-13T15:33:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmp41fz5juf/worker_log.txt +[2020-11-13T15:33:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmpocx70u6t/worker_log.txt +[2020-11-13T15:33:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmpc2a1y13_/worker_log.txt +[2020-11-13T15:33:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmphfv6vaot/worker_log.txt +[2020-11-13T15:33:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmphtqlxtnc/worker_log.txt +[2020-11-13T15:33:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmp5kfdmxb7/worker_log.txt +[2020-11-13T15:33:28-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-o1f0sosm +[2020-11-13T15:33:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:28: Obtaining file (mhci:method_file) to the file job store +[2020-11-13T15:33:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:28: Obtaining file (indexes:dbsnp_tbi) to the file job store +[2020-11-13T15:33:29-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5bdy2vh2 +[2020-11-13T15:33:32-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:32: Obtaining file (radia:gencode_beds) to the file job store +[2020-11-13T15:33:32-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:32: Obtaining file (indexes:cosmic_idx) to the file job store +[2020-11-13T15:33:32-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:32: Obtaining file (indexes:genome_fai) to the file job store +[2020-11-13T15:33:32-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:32: Obtaining file (mhcii:method_file) to the file job store +[2020-11-13T15:33:32-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:32: Obtaining file (radia:retrogene_beds) to the file job store +[2020-11-13T15:33:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:33: Obtaining file (reports:itx_resistance_file) to the file job store +[2020-11-13T15:33:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:33: Obtaining file (reports:car_t_targets_file) to the file job store +[2020-11-13T15:33:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:33: Obtaining file (indexes:genome_dict) to the file job store +[2020-11-13T15:33:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:33: Obtaining file (reports:immune_resistance_pathways_file) to the file job store +[2020-11-13T15:33:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:33: Obtaining file (strelka:config_file) to the file job store +[2020-11-13T15:33:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:33: Obtaining file (radia:pseudogene_beds) to the file job store +[2020-11-13T15:33:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:33: Obtaining file (indexes:dbsnp_idx) to the file job store +[2020-11-13T15:33:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:33: Obtaining file (reports:mhc_pathways_file) to the file job store +[2020-11-13T15:33:33-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-7temn04b +[2020-11-13T15:33:33-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1jrfy_07 +[2020-11-13T15:33:33-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5_jfkiu5 +[2020-11-13T15:33:33-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-4dyd2xg5 +[2020-11-13T15:33:33-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-mi83gk2v +[2020-11-13T15:33:33-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-40mnvizw +[2020-11-13T15:33:33-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-a481ugll +[2020-11-13T15:33:33-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-eg0mroe5 +[2020-11-13T15:33:33-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-09ig6r59 +[2020-11-13T15:33:33-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-a6ie8zcq +[2020-11-13T15:33:33-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-4etp2gh5 +[2020-11-13T15:33:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:33: Obtaining file (transgene:gencode_peptide_fasta) to the file job store +[2020-11-13T15:33:33-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-65blz2rt +[2020-11-13T15:33:33-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-9dkdlfhu +[2020-11-13T15:33:34-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:34: Obtaining file (transgene:gencode_transcript_fasta) to the file job store +[2020-11-13T15:33:34-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:34: Obtaining file (radia:cosmic_beds) to the file job store +[2020-11-13T15:33:34-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-nur3yi6b +[2020-11-13T15:33:34-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-wf_msw97 +[2020-11-13T15:33:34-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-4io4i1l8 +[2020-11-13T15:33:34-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-dqxfjdxj +[2020-11-13T15:33:34-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:34: Obtaining file (transgene:gencode_annotation_gtf) to the file job store +[2020-11-13T15:33:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:33:38: Obtaining file (indexes:cosmic_vcf) to the file job store +[2020-11-13T15:33:38-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-qac4y69v +[2020-11-13T15:34:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-01h704qy +[2020-11-13T15:34:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:34:40: Obtaining file (transgene:genome_fasta) to the file job store +[2020-11-13T15:34:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:34:40: Obtaining file (indexes:genome_fasta) to the file job store +[2020-11-13T15:34:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-diuz59sb +[2020-11-13T15:34:44-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:34:44: Obtaining file (radia:dbsnp_beds) to the file job store +[2020-11-13T15:34:44-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-2i61_7li +[2020-11-13T15:35:03-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-y1gvjtt_ +[2020-11-13T15:35:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:35:03: Obtaining file (snpeff:index) to the file job store +[2020-11-13T15:35:23-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-j6ge7umm +[2020-11-13T15:35:24-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:35:24: Obtaining file (rsem:index) to the file job store +[2020-11-13T15:36:44-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:36:44: Obtaining file (indexes:dbsnp_vcf) to the file job store +[2020-11-13T15:36:44-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-tgrluxoj +[2020-11-13T15:38:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:38:01: Obtaining file (bwa:index) to the file job store +[2020-11-13T15:38:02-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-zx6h9z5d +[2020-11-13T15:38:53-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5_apbcx6 +[2020-11-13T15:38:53-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:38:53: Obtaining file (phlat:index) to the file job store +[2020-11-13T15:56:29-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-jq4g_3yc +[2020-11-13T15:56:29-0800] [MainThread] [I] [toil.leader] Issued job 'launch_protect' kind-launch_protect/instance-jt487vo8 with job batch system ID: 30 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T15:56:30-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 15:56:30: Obtaining file (star:index) to the file job store +[2020-11-13T15:56:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmpravj4jvx/worker_log.txt +[2020-11-13T15:56:34-0800] [MainThread] [I] [toil.leader] Job ended: 'launch_protect' kind-launch_protect/instance-jt487vo8 +[2020-11-13T15:56:34-0800] [MainThread] [I] [toil.leader] Issued job 'prepare_samples' kind-prepare_samples/instance-pjtbswqv with job batch system ID: 31 and cores: 1, disk: 40.0 G, and memory: 2.0 G +[2020-11-13T15:56:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmpv9npzhxu/worker_log.txt +[2020-11-13T15:56:36-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker prepare_samples file:/scratch/drkthomp/jobStore kind-prepare_samples/instance-pjtbswqv. +[2020-11-13T15:56:36-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'prepare_samples' kind-prepare_samples/instance-pjtbswqv +[2020-11-13T15:56:36-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'prepare_samples' kind-prepare_samples/instance-pjtbswqv +[2020-11-13T15:56:36-0800] [MainThread] [W] [toil.leader] Log from job kind-prepare_samples/instance-pjtbswqv follows: +=========> + [2020-11-13T15:56:35-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-13T15:56:35-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-13T15:56:36-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Downloading Inputs for TEST + [2020-11-13T15:56:36-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (TEST:normal_dna_fastq_1) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 844, in prepare_samples + gdc_download_token=univ_options['gdc_download_token']) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 804, in get_pipeline_inputs + assert os.path.exists(input_file), 'Bogus Input : ' + input_file + AssertionError: Bogus Input : /scratch/drkthomp/gerald_D1VCPACXX_6_R1.fastq + [2020-11-13T15:56:36-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-13T15:56:36-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'prepare_samples' kind-prepare_samples/instance-pjtbswqv with ID kind-prepare_samples/instance-pjtbswqv to 1 +[2020-11-13T15:56:36-0800] [MainThread] [I] [toil.leader] Issued job 'prepare_samples' kind-prepare_samples/instance-pjtbswqv with job batch system ID: 32 and cores: 1, disk: 40.0 G, and memory: 2.0 G +[2020-11-13T15:56:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-1e404bf3-2ea9-43c9-8123-1155306f03c6-e8611b53-bdab-4598-acae-87977f99831e/tmpb4w00rr9/worker_log.txt +[2020-11-13T15:56:38-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker prepare_samples file:/scratch/drkthomp/jobStore kind-prepare_samples/instance-pjtbswqv. +[2020-11-13T15:56:38-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'prepare_samples' kind-prepare_samples/instance-pjtbswqv +[2020-11-13T15:56:38-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'prepare_samples' kind-prepare_samples/instance-pjtbswqv +[2020-11-13T15:56:38-0800] [MainThread] [W] [toil.leader] Log from job kind-prepare_samples/instance-pjtbswqv follows: +=========> + [2020-11-13T15:56:37-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-13T15:56:37-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-13T15:56:37-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Downloading Inputs for TEST + [2020-11-13T15:56:37-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (TEST:normal_dna_fastq_1) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 844, in prepare_samples + gdc_download_token=univ_options['gdc_download_token']) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 804, in get_pipeline_inputs + assert os.path.exists(input_file), 'Bogus Input : ' + input_file + AssertionError: Bogus Input : /scratch/drkthomp/gerald_D1VCPACXX_6_R1.fastq + [2020-11-13T15:56:37-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-13T15:56:38-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'prepare_samples' kind-prepare_samples/instance-pjtbswqv with ID kind-prepare_samples/instance-pjtbswqv to 0 +[2020-11-13T15:56:38-0800] [MainThread] [W] [toil.leader] Job 'prepare_samples' kind-prepare_samples/instance-pjtbswqv with ID kind-prepare_samples/instance-pjtbswqv is completely failed +[2020-11-13T15:56:46-0800] [MainThread] [I] [toil.leader] Finished toil run with 3 failed jobs. +[2020-11-13T15:56:46-0800] [MainThread] [I] [toil.leader] Failed jobs at end of the run: 'launch_protect' kind-launch_protect/instance-jt487vo8 'prepare_samples' kind-prepare_samples/instance-pjtbswqv 'get_all_tool_inputs' kind-parse_config_file/instance-qt4s63a_ +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/bin/ProTECT", line 11, in + load_entry_point('protect', 'console_scripts', 'ProTECT')() + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1025, in main + Job.Runner.startToil(start, params) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 796, in startToil + return toil.start(job) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 846, in start + return self._runMainLoop(rootJobGraph) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 1138, in _runMainLoop + jobCache=self._jobCache).run() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/leader.py", line 269, in run + raise FailedJobsException(self.config.jobStore, self.toilState.totalFailedJobs, self.jobStore) +toil.leader.FailedJobsException: The job store 'file:/scratch/drkthomp/jobStore' contains 3 failed jobs: 'launch_protect' kind-launch_protect/instance-jt487vo8, 'prepare_samples' kind-prepare_samples/instance-pjtbswqv, 'get_all_tool_inputs' kind-parse_config_file/instance-qt4s63a_ +Log from job 'prepare_samples' kind-prepare_samples/instance-pjtbswqv follows: +=========> + [2020-11-13T15:56:37-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-13T15:56:37-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-13T15:56:37-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Downloading Inputs for TEST + [2020-11-13T15:56:37-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (TEST:normal_dna_fastq_1) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 844, in prepare_samples + gdc_download_token=univ_options['gdc_download_token']) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 804, in get_pipeline_inputs + assert os.path.exists(input_file), 'Bogus Input : ' + input_file + AssertionError: Bogus Input : /scratch/drkthomp/gerald_D1VCPACXX_6_R1.fastq + [2020-11-13T15:56:37-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= diff --git a/errors/2020-11-13-16-34-54.txt b/errors/2020-11-13-16-34-54.txt new file mode 100644 index 00000000..35f783f3 --- /dev/null +++ b/errors/2020-11-13-16-34-54.txt @@ -0,0 +1,19 @@ +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 104, in initialize + os.mkdir(self.jobStoreDir) +FileExistsError: [Errno 17] File exists: '/scratch/drkthomp/jobStore' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/bin/ProTECT", line 11, in + load_entry_point('protect', 'console_scripts', 'ProTECT')() + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1025, in main + Job.Runner.startToil(start, params) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 794, in startToil + with Toil(options) as toil: + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 774, in __enter__ + jobStore.initialize(config) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 107, in initialize + raise JobStoreExistsException(self.jobStoreDir) +toil.jobStores.abstractJobStore.JobStoreExistsException: The job store '/scratch/drkthomp/jobStore' already exists. Use --restart to resume the workflow, or remove the job store with 'toil clean' to start the workflow from scratch. diff --git a/errors/2020-11-13-16-35-47.txt b/errors/2020-11-13-16-35-47.txt new file mode 100644 index 00000000..16701053 --- /dev/null +++ b/errors/2020-11-13-16-35-47.txt @@ -0,0 +1,252 @@ +[2020-11-13T16:35:49-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-13T16:35:49-0800] [MainThread] [I] [toil.leader] Issued job 'parse_config_file' kind-parse_config_file/instance-u5he9dsb with job batch system ID: 0 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpk1l9bkxn/worker_log.txt +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] 1 jobs are running, 0 jobs are issued and waiting to run +[2020-11-13T16:35:51-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:51: Parsing config file +[2020-11-13T16:35:51-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:51: Obtaining tool inputs +[2020-11-13T16:35:51-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:51: Obtained tool inputs +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Job ended: 'parse_config_file' kind-parse_config_file/instance-u5he9dsb +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-0y8sne0p with job batch system ID: 1 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-rr_ce0ks with job batch system ID: 2 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-c1hntyi9 with job batch system ID: 3 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-y5la78i4 with job batch system ID: 4 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-3g512kkl with job batch system ID: 5 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-l746xeyc with job batch system ID: 6 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-9n8w0pb9 with job batch system ID: 7 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ij6nieb0 with job batch system ID: 8 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-3eq_fdpb with job batch system ID: 9 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-f5cechio with job batch system ID: 10 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-q246byit with job batch system ID: 11 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ryk1vs7w with job batch system ID: 12 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-idxcdn5h with job batch system ID: 13 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-3hdnhh5w with job batch system ID: 14 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-qyxbv_i2 with job batch system ID: 15 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-4j4qilo9 with job batch system ID: 16 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-0avcq5ef with job batch system ID: 17 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-7xt8ohjl with job batch system ID: 18 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-qi6h1cpx with job batch system ID: 19 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-da72k6ae with job batch system ID: 20 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-dp6y6t0f with job batch system ID: 21 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-no06otk_ with job batch system ID: 22 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-gwixytds with job batch system ID: 23 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-jan3bh8c with job batch system ID: 24 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-j4vo873w with job batch system ID: 25 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-e3bk4_3o with job batch system ID: 26 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-w348il0n with job batch system ID: 27 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-9so22tsy with job batch system ID: 28 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-xnbsxg2z with job batch system ID: 29 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmp9jr3_7_s/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpr9t0rsdj/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpq69jy2eb/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpngtlve94/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmp5nlkr875/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpjwkriod2/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpzjaiw4ac/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmptede9fi5/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpldtacry1/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmppp9d2ve7/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpxwbxxwy4/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpe67hc61g/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmp64x3hlbp/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpy0o95bdy/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpwvvgpai4/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpdozc_w2u/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpzr5fh0hs/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmp48o2flor/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpl1vhp9_9/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmp792xcdri/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpe26vs8vu/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpe3w12vyz/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpd54bh8ck/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpqp7rpm_9/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpyecaucd1/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmp78pllp8w/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmp6orbgstp/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpr5yrnpq4/worker_log.txt +[2020-11-13T16:35:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmp4j0i0ucd/worker_log.txt +[2020-11-13T16:35:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:54: Obtaining file (indexes:cosmic_vcf) to the file job store +[2020-11-13T16:35:55-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-9n8w0pb9 +[2020-11-13T16:35:55-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ryk1vs7w +[2020-11-13T16:35:55-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-3g512kkl +[2020-11-13T16:35:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:55: Obtaining file (radia:cosmic_beds) to the file job store +[2020-11-13T16:35:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:55: Obtaining file (indexes:genome_fai) to the file job store +[2020-11-13T16:35:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:55: Obtaining file (indexes:dbsnp_idx) to the file job store +[2020-11-13T16:35:55-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-f5cechio +[2020-11-13T16:35:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:56: Obtaining file (indexes:cosmic_idx) to the file job store +[2020-11-13T16:35:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:56: Obtaining file (mhcii:method_file) to the file job store +[2020-11-13T16:35:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:56: Obtaining file (reports:car_t_targets_file) to the file job store +[2020-11-13T16:35:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:56: Obtaining file (indexes:dbsnp_tbi) to the file job store +[2020-11-13T16:35:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:56: Obtaining file (indexes:genome_fasta) to the file job store +[2020-11-13T16:35:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:56: Obtaining file (rsem:index) to the file job store +[2020-11-13T16:35:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:56: Obtaining file (reports:immune_resistance_pathways_file) to the file job store +[2020-11-13T16:35:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:56: Obtaining file (transgene:gencode_annotation_gtf) to the file job store +[2020-11-13T16:35:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:56: Obtaining file (transgene:gencode_peptide_fasta) to the file job store +[2020-11-13T16:35:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:56: Obtaining file (mhci:method_file) to the file job store +[2020-11-13T16:35:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:57: Obtaining file (radia:gencode_beds) to the file job store +[2020-11-13T16:35:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:57: Obtaining file (indexes:genome_dict) to the file job store +[2020-11-13T16:35:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:57: Obtaining file (strelka:config_file) to the file job store +[2020-11-13T16:35:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:57: Obtaining file (radia:retrogene_beds) to the file job store +[2020-11-13T16:35:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:57: Obtaining file (radia:pseudogene_beds) to the file job store +[2020-11-13T16:35:57-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ij6nieb0 +[2020-11-13T16:35:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:57: Obtaining file (reports:mhc_pathways_file) to the file job store +[2020-11-13T16:35:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:57: Obtaining file (reports:itx_resistance_file) to the file job store +[2020-11-13T16:35:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:57: Obtaining file (transgene:gencode_transcript_fasta) to the file job store +[2020-11-13T16:35:57-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-y5la78i4 +[2020-11-13T16:35:57-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-q246byit +[2020-11-13T16:35:57-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-c1hntyi9 +[2020-11-13T16:35:57-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-dp6y6t0f +[2020-11-13T16:35:57-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-jan3bh8c +[2020-11-13T16:35:57-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-qi6h1cpx +[2020-11-13T16:35:57-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-j4vo873w +[2020-11-13T16:35:57-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-9so22tsy +[2020-11-13T16:35:57-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-l746xeyc +[2020-11-13T16:35:57-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-4j4qilo9 +[2020-11-13T16:35:57-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-xnbsxg2z +[2020-11-13T16:35:57-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-3hdnhh5w +[2020-11-13T16:35:58-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-qyxbv_i2 +[2020-11-13T16:35:58-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-0avcq5ef +[2020-11-13T16:35:58-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-e3bk4_3o +[2020-11-13T16:35:58-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-w348il0n +[2020-11-13T16:35:58-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-da72k6ae +[2020-11-13T16:35:58-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-no06otk_ +[2020-11-13T16:35:58-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:58: Obtaining file (transgene:genome_fasta) to the file job store +[2020-11-13T16:35:58-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:58: Obtaining file (radia:dbsnp_beds) to the file job store +[2020-11-13T16:35:58-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-idxcdn5h +[2020-11-13T16:35:59-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-7xt8ohjl +[2020-11-13T16:35:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:59: Obtaining file (snpeff:index) to the file job store +[2020-11-13T16:35:59-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-3eq_fdpb +[2020-11-13T16:35:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:35:59: Obtaining file (indexes:dbsnp_vcf) to the file job store +[2020-11-13T16:36:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:36:00: Obtaining file (bwa:index) to the file job store +[2020-11-13T16:36:00-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-rr_ce0ks +[2020-11-13T16:36:02-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-gwixytds +[2020-11-13T16:36:02-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:36:02: Obtaining file (phlat:index) to the file job store +[2020-11-13T16:36:45-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-0y8sne0p +[2020-11-13T16:36:45-0800] [MainThread] [I] [toil.leader] Issued job 'launch_protect' kind-launch_protect/instance-2qmuo_xi with job batch system ID: 30 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-13T16:36:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-13-2020 16:36:45: Obtaining file (star:index) to the file job store +[2020-11-13T16:36:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpjsekj9us/worker_log.txt +[2020-11-13T16:37:32-0800] [MainThread] [I] [toil.leader] Job ended: 'launch_protect' kind-launch_protect/instance-2qmuo_xi +[2020-11-13T16:37:32-0800] [MainThread] [I] [toil.leader] Issued job 'prepare_samples' kind-prepare_samples/instance-jma2g4uf with job batch system ID: 31 and cores: 1, disk: 40.0 G, and memory: 2.0 G +[2020-11-13T16:37:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmp48xw92ho/worker_log.txt +[2020-11-13T17:11:27-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker prepare_samples file:/scratch/drkthomp/jobStore kind-prepare_samples/instance-jma2g4uf. +[2020-11-13T17:11:27-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'prepare_samples' kind-prepare_samples/instance-jma2g4uf +[2020-11-13T17:11:27-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'prepare_samples' kind-prepare_samples/instance-jma2g4uf +[2020-11-13T17:11:27-0800] [MainThread] [W] [toil.leader] Log from job kind-prepare_samples/instance-jma2g4uf follows: +=========> + [2020-11-13T16:37:32-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-13T16:37:32-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-13T16:37:34-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Downloading Inputs for TEST + [2020-11-13T16:37:34-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (TEST:tumor_dna_fastq_2) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 844, in prepare_samples + gdc_download_token=univ_options['gdc_download_token']) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 805, in get_pipeline_inputs + input_file = job.fileStore.writeGlobalFile(input_file) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/fileStores/nonCachingFileStore.py", line 91, in writeGlobalFile + fileStoreID = self.jobStore.writeFile(absLocalFileName, creatorID, cleanup) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 387, in writeFile + atomic_copy(localFilePath, absPath) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/lib/misc.py", line 225, in atomic_copy + shutil.copyfile(src_path, dest_path_tmp) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 122, in copyfile + copyfileobj(fsrc, fdst) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + OSError: [Errno 28] No space left on device + [2020-11-13T17:11:27-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-13T17:11:27-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'prepare_samples' kind-prepare_samples/instance-jma2g4uf with ID kind-prepare_samples/instance-jma2g4uf to 1 +[2020-11-13T17:11:27-0800] [MainThread] [I] [toil.leader] Issued job 'prepare_samples' kind-prepare_samples/instance-jma2g4uf with job batch system ID: 32 and cores: 1, disk: 40.0 G, and memory: 2.0 G +[2020-11-13T17:11:28-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-cca77788-8506-4ba2-af72-0e0b5dc8e1d6-e8611b53-bdab-4598-acae-87977f99831e/tmpjf5a_k8e/worker_log.txt +[2020-11-13T17:15:07-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker prepare_samples file:/scratch/drkthomp/jobStore kind-prepare_samples/instance-jma2g4uf. +[2020-11-13T17:15:07-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'prepare_samples' kind-prepare_samples/instance-jma2g4uf +[2020-11-13T17:15:07-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'prepare_samples' kind-prepare_samples/instance-jma2g4uf +[2020-11-13T17:15:07-0800] [MainThread] [W] [toil.leader] Log from job kind-prepare_samples/instance-jma2g4uf follows: +=========> + [2020-11-13T17:11:28-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-13T17:11:28-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-13T17:11:29-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Downloading Inputs for TEST + [2020-11-13T17:11:29-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (TEST:tumor_dna_fastq_2) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 844, in prepare_samples + gdc_download_token=univ_options['gdc_download_token']) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 805, in get_pipeline_inputs + input_file = job.fileStore.writeGlobalFile(input_file) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/fileStores/nonCachingFileStore.py", line 91, in writeGlobalFile + fileStoreID = self.jobStore.writeFile(absLocalFileName, creatorID, cleanup) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 387, in writeFile + atomic_copy(localFilePath, absPath) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/lib/misc.py", line 225, in atomic_copy + shutil.copyfile(src_path, dest_path_tmp) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 122, in copyfile + copyfileobj(fsrc, fdst) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + OSError: [Errno 28] No space left on device + [2020-11-13T17:15:07-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-13T17:15:07-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'prepare_samples' kind-prepare_samples/instance-jma2g4uf with ID kind-prepare_samples/instance-jma2g4uf to 0 +[2020-11-13T17:15:07-0800] [MainThread] [W] [toil.leader] Job 'prepare_samples' kind-prepare_samples/instance-jma2g4uf with ID kind-prepare_samples/instance-jma2g4uf is completely failed +[2020-11-13T17:15:16-0800] [MainThread] [I] [toil.leader] Finished toil run with 3 failed jobs. +[2020-11-13T17:15:16-0800] [MainThread] [I] [toil.leader] Failed jobs at end of the run: 'get_all_tool_inputs' kind-parse_config_file/instance-u5he9dsb 'launch_protect' kind-launch_protect/instance-2qmuo_xi 'prepare_samples' kind-prepare_samples/instance-jma2g4uf +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/bin/ProTECT", line 11, in + load_entry_point('protect', 'console_scripts', 'ProTECT')() + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1025, in main + Job.Runner.startToil(start, params) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 796, in startToil + return toil.start(job) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 846, in start + return self._runMainLoop(rootJobGraph) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 1138, in _runMainLoop + jobCache=self._jobCache).run() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/leader.py", line 269, in run + raise FailedJobsException(self.config.jobStore, self.toilState.totalFailedJobs, self.jobStore) +toil.leader.FailedJobsException: The job store 'file:/scratch/drkthomp/jobStore' contains 3 failed jobs: 'get_all_tool_inputs' kind-parse_config_file/instance-u5he9dsb, 'launch_protect' kind-launch_protect/instance-2qmuo_xi, 'prepare_samples' kind-prepare_samples/instance-jma2g4uf +Log from job 'prepare_samples' kind-prepare_samples/instance-jma2g4uf follows: +=========> + [2020-11-13T17:11:28-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-13T17:11:28-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-13T17:11:29-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Downloading Inputs for TEST + [2020-11-13T17:11:29-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Obtaining file (TEST:tumor_dna_fastq_2) to the file job store + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 844, in prepare_samples + gdc_download_token=univ_options['gdc_download_token']) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 805, in get_pipeline_inputs + input_file = job.fileStore.writeGlobalFile(input_file) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/fileStores/nonCachingFileStore.py", line 91, in writeGlobalFile + fileStoreID = self.jobStore.writeFile(absLocalFileName, creatorID, cleanup) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 387, in writeFile + atomic_copy(localFilePath, absPath) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/lib/misc.py", line 225, in atomic_copy + shutil.copyfile(src_path, dest_path_tmp) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 122, in copyfile + copyfileobj(fsrc, fdst) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + OSError: [Errno 28] No space left on device + [2020-11-13T17:15:07-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= diff --git a/errors/2020-11-15-00-28-09.txt b/errors/2020-11-15-00-28-09.txt new file mode 100644 index 00000000..87a07371 --- /dev/null +++ b/errors/2020-11-15-00-28-09.txt @@ -0,0 +1,720 @@ +[2020-11-15T00:28:10-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-15T00:28:11-0800] [MainThread] [I] [toil.leader] Issued job 'parse_config_file' kind-parse_config_file/instance-ml5szru_ with job batch system ID: 0 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmppxo7wk8d/worker_log.txt +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Job ended: 'parse_config_file' kind-parse_config_file/instance-ml5szru_ +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] 0 jobs are running, 0 jobs are issued and waiting to run +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-xlca_kav with job batch system ID: 1 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-w0r0fgkl with job batch system ID: 2 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-4449i24l with job batch system ID: 3 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-lwawpwto with job batch system ID: 4 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-moo53zz6 with job batch system ID: 5 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-s36dw4ty with job batch system ID: 6 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-_5sx0yzj with job batch system ID: 7 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-_91ux51r with job batch system ID: 8 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-e60iam16 with job batch system ID: 9 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-kruxi_g4 with job batch system ID: 10 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1y_velvd with job batch system ID: 11 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-9yk9icyt with job batch system ID: 12 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-nb2ttglx with job batch system ID: 13 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-k53dxsrv with job batch system ID: 14 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1jy08oj3 with job batch system ID: 15 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-fp_69yov with job batch system ID: 16 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1iu44e5t with job batch system ID: 17 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-q7rykuf9 with job batch system ID: 18 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-28832prp with job batch system ID: 19 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-v9w5g4zj with job batch system ID: 20 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-g34bgkh6 with job batch system ID: 21 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-4okw3zyq with job batch system ID: 22 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-igauxjaw with job batch system ID: 23 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-mjjj9pl3 with job batch system ID: 24 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-fdqruxwl with job batch system ID: 25 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-gquoa6eo with job batch system ID: 26 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-b07a33j5 with job batch system ID: 27 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-4wagfh4p with job batch system ID: 28 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1i_zjhkv with job batch system ID: 29 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:28:13-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:13: Parsing config file +[2020-11-15T00:28:13-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:13: Obtaining tool inputs +[2020-11-15T00:28:13-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:13: Obtained tool inputs +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpvncllpoc/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpm8ckh0gz/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp3g54ebyq/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp873znnf0/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpv414u8qc/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmppwo8qx_x/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpt4x6mc8d/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp26d70kcl/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpwrp008xn/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpqkw2t5jc/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp4o3681u6/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp_jplrlq7/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpje66eddv/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp4p00aryl/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp4m_504_n/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp3u5y9dn7/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp3yqbr8f1/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpcf10i63i/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpnsnvn10s/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp39kl6szq/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpolc0_9s3/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp1342g8o9/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpnjhlz_m_/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp5o4zjpxi/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpemqjsf_7/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp56aktw6z/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmps71f55cu/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp9v0ir0_w/worker_log.txt +[2020-11-15T00:28:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpwybr_mle/worker_log.txt +[2020-11-15T00:28:15-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-s36dw4ty +[2020-11-15T00:28:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:16: Obtaining file (indexes:genome_dict) to the file job store +[2020-11-15T00:28:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:16: Obtaining file (indexes:genome_fai) to the file job store +[2020-11-15T00:28:16-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-moo53zz6 +[2020-11-15T00:28:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:16: Obtaining file (radia:retrogene_beds) to the file job store +[2020-11-15T00:28:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:16: Obtaining file (radia:pseudogene_beds) to the file job store +[2020-11-15T00:28:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:17: Obtaining file (indexes:cosmic_idx) to the file job store +[2020-11-15T00:28:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:17: Obtaining file (reports:immune_resistance_pathways_file) to the file job store +[2020-11-15T00:28:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:17: Obtaining file (indexes:dbsnp_tbi) to the file job store +[2020-11-15T00:28:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:17: Obtaining file (reports:itx_resistance_file) to the file job store +[2020-11-15T00:28:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:17: Obtaining file (reports:mhc_pathways_file) to the file job store +[2020-11-15T00:28:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:17: Obtaining file (strelka:config_file) to the file job store +[2020-11-15T00:28:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:17: Obtaining file (radia:gencode_beds) to the file job store +[2020-11-15T00:28:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:17: Obtaining file (indexes:dbsnp_idx) to the file job store +[2020-11-15T00:28:17-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1jy08oj3 +[2020-11-15T00:28:18-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-k53dxsrv +[2020-11-15T00:28:18-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:18: Obtaining file (mhci:method_file) to the file job store +[2020-11-15T00:28:18-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:18: Obtaining file (mhcii:method_file) to the file job store +[2020-11-15T00:28:18-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:18: Obtaining file (radia:cosmic_beds) to the file job store +[2020-11-15T00:28:18-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:18: Obtaining file (reports:car_t_targets_file) to the file job store +[2020-11-15T00:28:18-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-_91ux51r +[2020-11-15T00:28:18-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-4wagfh4p +[2020-11-15T00:28:18-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1y_velvd +[2020-11-15T00:28:18-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-b07a33j5 +[2020-11-15T00:28:18-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-gquoa6eo +[2020-11-15T00:28:18-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-kruxi_g4 +[2020-11-15T00:28:18-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-fp_69yov +[2020-11-15T00:28:18-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1iu44e5t +[2020-11-15T00:28:18-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-mjjj9pl3 +[2020-11-15T00:28:18-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-9yk9icyt +[2020-11-15T00:28:18-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-fdqruxwl +[2020-11-15T00:28:18-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1i_zjhkv +[2020-11-15T00:28:18-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-_5sx0yzj +[2020-11-15T00:28:18-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:18: Obtaining file (indexes:cosmic_vcf) to the file job store +[2020-11-15T00:28:18-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:18: Obtaining file (transgene:gencode_peptide_fasta) to the file job store +[2020-11-15T00:28:18-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-28832prp +[2020-11-15T00:28:19-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-v9w5g4zj +[2020-11-15T00:28:19-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:19: Obtaining file (transgene:gencode_transcript_fasta) to the file job store +[2020-11-15T00:28:20-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-g34bgkh6 +[2020-11-15T00:28:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:20: Obtaining file (transgene:gencode_annotation_gtf) to the file job store +[2020-11-15T00:28:42-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-lwawpwto +[2020-11-15T00:28:43-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-4okw3zyq +[2020-11-15T00:28:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:43: Obtaining file (indexes:genome_fasta) to the file job store +[2020-11-15T00:28:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:43: Obtaining file (transgene:genome_fasta) to the file job store +[2020-11-15T00:28:44-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-nb2ttglx +[2020-11-15T00:28:44-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:28:44: Obtaining file (radia:dbsnp_beds) to the file job store +[2020-11-15T00:29:04-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:29:04: Obtaining file (snpeff:index) to the file job store +[2020-11-15T00:29:04-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:29:04: Obtaining file (rsem:index) to the file job store +[2020-11-15T00:29:04-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-q7rykuf9 +[2020-11-15T00:29:04-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-4449i24l +[2020-11-15T00:29:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:29:26: Obtaining file (indexes:dbsnp_vcf) to the file job store +[2020-11-15T00:29:26-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-e60iam16 +[2020-11-15T00:30:07-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-w0r0fgkl +[2020-11-15T00:30:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:30:07: Obtaining file (bwa:index) to the file job store +[2020-11-15T00:30:15-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-igauxjaw +[2020-11-15T00:30:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:30:15: Obtaining file (phlat:index) to the file job store +[2020-11-15T00:32:10-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-xlca_kav +[2020-11-15T00:32:10-0800] [MainThread] [I] [toil.leader] Issued job 'launch_protect' kind-launch_protect/instance-9wa2ilmx with job batch system ID: 30 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:32:11-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:32:11: Obtaining file (star:index) to the file job store +[2020-11-15T00:32:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp55ch8a3g/worker_log.txt +[2020-11-15T00:32:13-0800] [MainThread] [I] [toil.leader] Job ended: 'launch_protect' kind-launch_protect/instance-9wa2ilmx +[2020-11-15T00:32:13-0800] [MainThread] [I] [toil.leader] Issued job 'prepare_samples' kind-prepare_samples/instance-melw0o77 with job batch system ID: 31 and cores: 1, disk: 40.0 G, and memory: 2.0 G +[2020-11-15T00:32:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpbbm40ixo/worker_log.txt +[2020-11-15T00:32:18-0800] [MainThread] [I] [toil.leader] Job ended: 'prepare_samples' kind-prepare_samples/instance-melw0o77 +[2020-11-15T00:32:18-0800] [MainThread] [I] [toil.leader] Issued job 'get_patient_fastqs' kind-get_patient_fastqs/instance-z2akd685 with job batch system ID: 32 and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-11-15T00:32:18-0800] [MainThread] [I] [toil.leader] Issued job 'get_patient_fastqs' kind-get_patient_fastqs/instance-3b6z2ec2 with job batch system ID: 33 and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-11-15T00:32:18-0800] [MainThread] [I] [toil.leader] Issued job 'get_patient_fastqs' kind-get_patient_fastqs/instance-g31njymf with job batch system ID: 34 and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-11-15T00:32:18-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:32:18: Downloading Inputs for TEST +[2020-11-15T00:32:18-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:32:18: Obtaining file (TEST:normal_dna_fastq_2) to the file job store +[2020-11-15T00:32:18-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:32:18: Obtaining file (TEST:tumor_rna_fastq_1) to the file job store +[2020-11-15T00:32:18-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:32:18: Obtaining file (TEST:normal_dna_fastq_1) to the file job store +[2020-11-15T00:32:18-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:32:18: Obtaining file (TEST:tumor_rna_fastq_2) to the file job store +[2020-11-15T00:32:18-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:32:18: Obtaining file (TEST:tumor_dna_fastq_2) to the file job store +[2020-11-15T00:32:18-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-15-2020 00:32:18: Obtaining file (TEST:tumor_dna_fastq_1) to the file job store +[2020-11-15T00:32:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpbfdfb38q/worker_log.txt +[2020-11-15T00:32:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp0wkkhiwy/worker_log.txt +[2020-11-15T00:32:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpwqv62_9x/worker_log.txt +[2020-11-15T00:32:20-0800] [MainThread] [I] [toil.leader] Job ended: 'get_patient_fastqs' kind-get_patient_fastqs/instance-z2akd685 +[2020-11-15T00:32:20-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-6r5wxjaz with job batch system ID: 35 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-15T00:32:20-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-w4x4xjcb with job batch system ID: 36 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-15T00:32:20-0800] [MainThread] [I] [toil.leader] Job ended: 'get_patient_fastqs' kind-get_patient_fastqs/instance-g31njymf +[2020-11-15T00:32:20-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-0l8ktz2m with job batch system ID: 37 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-15T00:32:20-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-9fqlkbpn with job batch system ID: 38 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-15T00:32:20-0800] [MainThread] [I] [toil.leader] Job ended: 'get_patient_fastqs' kind-get_patient_fastqs/instance-3b6z2ec2 +[2020-11-15T00:32:20-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-gxsdjbsw with job batch system ID: 39 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-15T00:32:20-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-fg9lg8z2 with job batch system ID: 40 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-15T00:32:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpnxt8xcij/worker_log.txt +[2020-11-15T00:32:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpihta2mkm/worker_log.txt +[2020-11-15T00:32:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp548sakwu/worker_log.txt +[2020-11-15T00:32:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp5nsvjnk7/worker_log.txt +[2020-11-15T00:32:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpag4u9ej2/worker_log.txt +[2020-11-15T00:32:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpve4br74u/worker_log.txt +[2020-11-15T00:32:21-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-6r5wxjaz +[2020-11-15T00:32:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-3chuyae2 with job batch system ID: 41 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-15T00:32:22-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-0l8ktz2m +[2020-11-15T00:32:22-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-rssn99a0 with job batch system ID: 42 and cores: 80, disk: 8.2 G, and memory: 2.0 G +[2020-11-15T00:32:22-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-gxsdjbsw +[2020-11-15T00:32:22-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-z35qoqn5 with job batch system ID: 43 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-15T00:32:22-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-w4x4xjcb +[2020-11-15T00:32:22-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-bou2djh0 with job batch system ID: 44 and cores: 80, disk: 13.4 G, and memory: 12.0 G +[2020-11-15T00:32:22-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-9fqlkbpn +[2020-11-15T00:32:22-0800] [MainThread] [I] [toil.leader] Issued job 'run_cutadapt' kind-run_cutadapt/instance-zdc0z58x with job batch system ID: 45 and cores: 1, disk: 1.8 G, and memory: 2.0 G +[2020-11-15T00:32:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmphy_rjfhn/worker_log.txt +[2020-11-15T00:32:22-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-fg9lg8z2 +[2020-11-15T00:32:22-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-fvcm11nr with job batch system ID: 46 and cores: 80, disk: 12.8 G, and memory: 12.0 G +[2020-11-15T00:32:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp4_ttlnyx/worker_log.txt +[2020-11-15T00:33:53-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_cutadapt file:/scratch/drkthomp/jobStore kind-run_cutadapt/instance-zdc0z58x. +[2020-11-15T00:33:53-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_cutadapt' kind-run_cutadapt/instance-zdc0z58x +[2020-11-15T00:33:53-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_cutadapt' kind-run_cutadapt/instance-zdc0z58x +[2020-11-15T00:33:53-0800] [MainThread] [W] [toil.leader] Log from job kind-run_cutadapt/instance-zdc0z58x follows: +=========> + [2020-11-15T00:32:22-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-15T00:32:22-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/qc/rna.py", line 59, in run_cutadapt + dockerhub=univ_options['dockerhub'], tool_version=cutadapt_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-15T00:33:53-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-15T00:33:53-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_cutadapt' kind-run_cutadapt/instance-zdc0z58x with ID kind-run_cutadapt/instance-zdc0z58x to 1 +[2020-11-15T00:33:53-0800] [MainThread] [W] [toil.jobGraph] We have increased the disk of the failed job 'run_cutadapt' kind-run_cutadapt/instance-zdc0z58x to the default of 2147483648 bytes +[2020-11-15T00:33:53-0800] [MainThread] [I] [toil.leader] Issued job 'run_cutadapt' kind-run_cutadapt/instance-zdc0z58x with job batch system ID: 47 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-15T00:34:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpzl1rm73q/worker_log.txt +[2020-11-15T00:47:35-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-3chuyae2. +[2020-11-15T00:47:35-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-3chuyae2 +[2020-11-15T00:47:35-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-3chuyae2 +[2020-11-15T00:47:35-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-3chuyae2 follows: +=========> + [2020-11-15T00:32:22-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-15T00:32:22-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-15T00:47:31-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmphy_rjfhn/446e9b13-3d2b-49c2-b442-7bb98ea88f41/input_1.fastq', 'input_2.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmphy_rjfhn/446e9b13-3d2b-49c2-b442-7bb98ea88f41/input_2.fastq', 'phlat_index.tar.gz': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmphy_rjfhn/446e9b13-3d2b-49c2-b442-7bb98ea88f41/phlat_index.tar.gz'} +<========= +[2020-11-15T00:47:35-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-3chuyae2 with ID kind-run_phlat/instance-3chuyae2 to 1 +[2020-11-15T00:47:35-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-3chuyae2 with job batch system ID: 48 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-15T00:47:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpe462tvp3/worker_log.txt +[2020-11-15T00:49:41-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_bwa file:/scratch/drkthomp/jobStore kind-run_bwa/instance-fvcm11nr. +[2020-11-15T00:49:41-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_bwa' kind-run_bwa/instance-fvcm11nr +[2020-11-15T00:49:41-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_bwa' kind-run_bwa/instance-fvcm11nr +[2020-11-15T00:49:41-0800] [MainThread] [W] [toil.leader] Log from job kind-run_bwa/instance-fvcm11nr follows: +=========> + [2020-11-15T00:36:49-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-15T00:36:49-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/alignment/dna.py", line 142, in run_bwa + tool_version=bwa_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-15T00:49:36-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-15T00:49:41-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_bwa' kind-run_bwa/instance-fvcm11nr with ID kind-run_bwa/instance-fvcm11nr to 1 +[2020-11-15T00:49:41-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-fvcm11nr with job batch system ID: 49 and cores: 80, disk: 12.8 G, and memory: 12.0 G +[2020-11-15T00:49:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpkc9r42n6/worker_log.txt +[2020-11-15T01:05:00-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-z35qoqn5. +[2020-11-15T01:05:00-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-z35qoqn5 +[2020-11-15T01:05:00-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-z35qoqn5 +[2020-11-15T01:05:00-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-z35qoqn5 follows: +=========> + [2020-11-15T00:49:41-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-15T00:49:41-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + [2020-11-15T01:04:56-0800] [MainThread] [W] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-gxsdjbsw/cleanup/file-m5btl4fd/stream used 100.26% (8.3 GB [8937328640B] used, 8.3 GB [8914445280B] requested) at the end of its run. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-15T01:04:56-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpkc9r42n6/dd78e603-aed3-4dff-93b0-12359cb8a8df/input_1.fastq', 'input_2.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpkc9r42n6/dd78e603-aed3-4dff-93b0-12359cb8a8df/input_2.fastq', 'phlat_index.tar.gz': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpkc9r42n6/dd78e603-aed3-4dff-93b0-12359cb8a8df/phlat_index.tar.gz'} +<========= +[2020-11-15T01:05:00-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-z35qoqn5 with ID kind-run_phlat/instance-z35qoqn5 to 1 +[2020-11-15T01:05:00-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-z35qoqn5 with job batch system ID: 50 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-15T01:05:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp_0lelddo/worker_log.txt +[2020-11-15T01:06:37-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-rssn99a0. +[2020-11-15T01:06:37-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-rssn99a0 +[2020-11-15T01:06:37-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-rssn99a0 +[2020-11-15T01:06:37-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-rssn99a0 follows: +=========> + [2020-11-15T00:47:35-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-15T00:47:35-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-15T01:06:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpe462tvp3/6525744e-5cc8-4261-93e3-ea8705ff5bea/input_1.fastq', 'input_2.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpe462tvp3/6525744e-5cc8-4261-93e3-ea8705ff5bea/input_2.fastq', 'phlat_index.tar.gz': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpe462tvp3/6525744e-5cc8-4261-93e3-ea8705ff5bea/phlat_index.tar.gz'} +<========= +[2020-11-15T01:06:37-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-rssn99a0 with ID kind-run_phlat/instance-rssn99a0 to 1 +[2020-11-15T01:06:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-rssn99a0 with job batch system ID: 51 and cores: 80, disk: 8.2 G, and memory: 2.0 G +[2020-11-15T01:06:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp_06vhw95/worker_log.txt +[2020-11-15T01:19:35-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-3chuyae2. +[2020-11-15T01:19:35-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-3chuyae2 +[2020-11-15T01:19:35-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-3chuyae2 +[2020-11-15T01:19:35-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-3chuyae2 follows: +=========> + [2020-11-15T01:05:40-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-15T01:05:40-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-15T01:19:31-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp_0lelddo/56082201-c209-4966-aac9-416a6c9f8418/input_1.fastq', 'input_2.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp_0lelddo/56082201-c209-4966-aac9-416a6c9f8418/input_2.fastq', 'phlat_index.tar.gz': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp_0lelddo/56082201-c209-4966-aac9-416a6c9f8418/phlat_index.tar.gz'} +<========= +[2020-11-15T01:19:35-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-3chuyae2 with ID kind-run_phlat/instance-3chuyae2 to 0 +[2020-11-15T01:19:35-0800] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-3chuyae2 with ID kind-run_phlat/instance-3chuyae2 is completely failed +[2020-11-15T01:19:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpy_dkrea9/worker_log.txt +[2020-11-15T01:20:52-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_bwa file:/scratch/drkthomp/jobStore kind-run_bwa/instance-fvcm11nr. +[2020-11-15T01:20:52-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_bwa' kind-run_bwa/instance-fvcm11nr +[2020-11-15T01:20:52-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_bwa' kind-run_bwa/instance-fvcm11nr +[2020-11-15T01:20:52-0800] [MainThread] [W] [toil.leader] Log from job kind-run_bwa/instance-fvcm11nr follows: +=========> + [2020-11-15T01:06:38-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-15T01:06:38-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/alignment/dna.py", line 142, in run_bwa + tool_version=bwa_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-15T01:20:32-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-15T01:20:52-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_bwa' kind-run_bwa/instance-fvcm11nr with ID kind-run_bwa/instance-fvcm11nr to 0 +[2020-11-15T01:20:52-0800] [MainThread] [W] [toil.leader] Job 'run_bwa' kind-run_bwa/instance-fvcm11nr with ID kind-run_bwa/instance-fvcm11nr is completely failed +[2020-11-15T01:20:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpuu1i8kep/worker_log.txt +[2020-11-15T01:28:14-0800] [MainThread] [I] [toil.leader] 2 jobs are running, 2 jobs are issued and waiting to run +[2020-11-15T01:35:01-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-rssn99a0. +[2020-11-15T01:35:01-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-rssn99a0 +[2020-11-15T01:35:01-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-rssn99a0 +[2020-11-15T01:35:01-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-rssn99a0 follows: +=========> + [2020-11-15T01:19:36-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-15T01:19:36-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-15T01:34:56-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpy_dkrea9/78036fa8-a915-466e-aaef-c8d1f08c7bd7/input_1.fastq', 'input_2.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpy_dkrea9/78036fa8-a915-466e-aaef-c8d1f08c7bd7/input_2.fastq', 'phlat_index.tar.gz': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpy_dkrea9/78036fa8-a915-466e-aaef-c8d1f08c7bd7/phlat_index.tar.gz'} +<========= +[2020-11-15T01:35:01-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-rssn99a0 with ID kind-run_phlat/instance-rssn99a0 to 0 +[2020-11-15T01:35:01-0800] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-rssn99a0 with ID kind-run_phlat/instance-rssn99a0 is completely failed +[2020-11-15T01:35:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpf8x9yyk2/worker_log.txt +[2020-11-15T01:36:31-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_bwa file:/scratch/drkthomp/jobStore kind-run_bwa/instance-bou2djh0. +[2020-11-15T01:36:31-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_bwa' kind-run_bwa/instance-bou2djh0 +[2020-11-15T01:36:31-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_bwa' kind-run_bwa/instance-bou2djh0 +[2020-11-15T01:36:31-0800] [MainThread] [W] [toil.leader] Log from job kind-run_bwa/instance-bou2djh0 follows: +=========> + [2020-11-15T01:20:53-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-15T01:20:53-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/alignment/dna.py", line 142, in run_bwa + tool_version=bwa_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-15T01:36:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-15T01:36:31-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_bwa' kind-run_bwa/instance-bou2djh0 with ID kind-run_bwa/instance-bou2djh0 to 1 +[2020-11-15T01:36:31-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-bou2djh0 with job batch system ID: 52 and cores: 80, disk: 13.4 G, and memory: 12.0 G +[2020-11-15T01:36:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpoq63s6mq/worker_log.txt +[2020-11-15T01:40:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_cutadapt file:/scratch/drkthomp/jobStore kind-run_cutadapt/instance-zdc0z58x. +[2020-11-15T01:40:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_cutadapt' kind-run_cutadapt/instance-zdc0z58x +[2020-11-15T01:40:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_cutadapt' kind-run_cutadapt/instance-zdc0z58x +[2020-11-15T01:40:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_cutadapt/instance-zdc0z58x follows: +=========> + [2020-11-15T01:36:32-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-15T01:36:32-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/qc/rna.py", line 59, in run_cutadapt + dockerhub=univ_options['dockerhub'], tool_version=cutadapt_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-15T01:40:09-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-15T01:40:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_cutadapt' kind-run_cutadapt/instance-zdc0z58x with ID kind-run_cutadapt/instance-zdc0z58x to 0 +[2020-11-15T01:40:10-0800] [MainThread] [W] [toil.leader] Job 'run_cutadapt' kind-run_cutadapt/instance-zdc0z58x with ID kind-run_cutadapt/instance-zdc0z58x is completely failed +[2020-11-15T01:40:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpd87eiccp/worker_log.txt +[2020-11-15T01:50:51-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-z35qoqn5. +[2020-11-15T01:50:51-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-z35qoqn5 +[2020-11-15T01:50:51-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-z35qoqn5 +[2020-11-15T01:50:51-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-z35qoqn5 follows: +=========> + [2020-11-15T01:35:02-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-15T01:35:02-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-15T01:50:47-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpf8x9yyk2/1454df9f-4b67-4586-b6d7-e59750b2b923/input_1.fastq', 'input_2.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpf8x9yyk2/1454df9f-4b67-4586-b6d7-e59750b2b923/input_2.fastq', 'phlat_index.tar.gz': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpf8x9yyk2/1454df9f-4b67-4586-b6d7-e59750b2b923/phlat_index.tar.gz'} +<========= +[2020-11-15T01:50:51-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-z35qoqn5 with ID kind-run_phlat/instance-z35qoqn5 to 0 +[2020-11-15T01:50:51-0800] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-z35qoqn5 with ID kind-run_phlat/instance-z35qoqn5 is completely failed +[2020-11-15T01:51:36-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_bwa file:/scratch/drkthomp/jobStore kind-run_bwa/instance-bou2djh0. +[2020-11-15T01:51:36-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_bwa' kind-run_bwa/instance-bou2djh0 +[2020-11-15T01:51:36-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_bwa' kind-run_bwa/instance-bou2djh0 +[2020-11-15T01:51:36-0800] [MainThread] [W] [toil.leader] Log from job kind-run_bwa/instance-bou2djh0 follows: +=========> + [2020-11-15T01:40:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-15T01:40:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/alignment/dna.py", line 142, in run_bwa + tool_version=bwa_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-15T01:51:31-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-15T01:51:36-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_bwa' kind-run_bwa/instance-bou2djh0 with ID kind-run_bwa/instance-bou2djh0 to 0 +[2020-11-15T01:51:36-0800] [MainThread] [W] [toil.leader] Job 'run_bwa' kind-run_bwa/instance-bou2djh0 with ID kind-run_bwa/instance-bou2djh0 is completely failed +[2020-11-15T01:51:50-0800] [MainThread] [I] [toil.leader] Finished toil run with 18 failed jobs. +[2020-11-15T01:51:50-0800] [MainThread] [I] [toil.leader] Failed jobs at end of the run: 'get_all_tool_inputs' kind-parse_config_file/instance-ml5szru_ 'run_bwa' kind-EncapsulatedJob/instance-fg9lg8z2 'get_patient_fastqs' kind-get_patient_fastqs/instance-g31njymf 'run_bwa' kind-EncapsulatedJob/instance-w4x4xjcb 'run_phlat' kind-run_phlat/instance-3chuyae2 'run_phlat' kind-EncapsulatedJob/instance-gxsdjbsw 'run_bwa' kind-run_bwa/instance-bou2djh0 'run_phlat' kind-run_phlat/instance-rssn99a0 'get_patient_fastqs' kind-get_patient_fastqs/instance-z2akd685 'prepare_samples' kind-prepare_samples/instance-melw0o77 'launch_protect' kind-launch_protect/instance-9wa2ilmx 'run_phlat' kind-EncapsulatedJob/instance-6r5wxjaz 'run_bwa' kind-run_bwa/instance-fvcm11nr 'run_phlat' kind-EncapsulatedJob/instance-0l8ktz2m 'run_phlat' kind-run_phlat/instance-z35qoqn5 'run_cutadapt' kind-run_cutadapt/instance-zdc0z58x 'get_patient_fastqs' kind-get_patient_fastqs/instance-3b6z2ec2 'run_cutadapt' kind-EncapsulatedJob/instance-9fqlkbpn +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/bin/ProTECT", line 11, in + load_entry_point('protect', 'console_scripts', 'ProTECT')() + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1025, in main + Job.Runner.startToil(start, params) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 796, in startToil + return toil.start(job) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 846, in start + return self._runMainLoop(rootJobGraph) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 1138, in _runMainLoop + jobCache=self._jobCache).run() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/leader.py", line 269, in run + raise FailedJobsException(self.config.jobStore, self.toilState.totalFailedJobs, self.jobStore) +toil.leader.FailedJobsException: The job store 'file:/scratch/drkthomp/jobStore' contains 18 failed jobs: 'get_all_tool_inputs' kind-parse_config_file/instance-ml5szru_, 'run_bwa' kind-EncapsulatedJob/instance-fg9lg8z2, 'get_patient_fastqs' kind-get_patient_fastqs/instance-g31njymf, 'run_bwa' kind-EncapsulatedJob/instance-w4x4xjcb, 'run_phlat' kind-run_phlat/instance-3chuyae2, 'run_phlat' kind-EncapsulatedJob/instance-gxsdjbsw, 'run_bwa' kind-run_bwa/instance-bou2djh0, 'run_phlat' kind-run_phlat/instance-rssn99a0, 'get_patient_fastqs' kind-get_patient_fastqs/instance-z2akd685, 'prepare_samples' kind-prepare_samples/instance-melw0o77, 'launch_protect' kind-launch_protect/instance-9wa2ilmx, 'run_phlat' kind-EncapsulatedJob/instance-6r5wxjaz, 'run_bwa' kind-run_bwa/instance-fvcm11nr, 'run_phlat' kind-EncapsulatedJob/instance-0l8ktz2m, 'run_phlat' kind-run_phlat/instance-z35qoqn5, 'run_cutadapt' kind-run_cutadapt/instance-zdc0z58x, 'get_patient_fastqs' kind-get_patient_fastqs/instance-3b6z2ec2, 'run_cutadapt' kind-EncapsulatedJob/instance-9fqlkbpn +Log from job 'run_phlat' kind-run_phlat/instance-3chuyae2 follows: +=========> + [2020-11-15T01:05:40-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-15T01:05:40-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-15T01:19:31-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp_0lelddo/56082201-c209-4966-aac9-416a6c9f8418/input_1.fastq', 'input_2.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp_0lelddo/56082201-c209-4966-aac9-416a6c9f8418/input_2.fastq', 'phlat_index.tar.gz': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp_0lelddo/56082201-c209-4966-aac9-416a6c9f8418/phlat_index.tar.gz'} +<========= +Log from job 'run_bwa' kind-run_bwa/instance-bou2djh0 follows: +=========> + [2020-11-15T01:40:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-15T01:40:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/alignment/dna.py", line 142, in run_bwa + tool_version=bwa_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-15T01:51:31-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +Log from job 'run_phlat' kind-run_phlat/instance-rssn99a0 follows: +=========> + [2020-11-15T01:19:36-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-15T01:19:36-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-15T01:34:56-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpy_dkrea9/78036fa8-a915-466e-aaef-c8d1f08c7bd7/input_1.fastq', 'input_2.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpy_dkrea9/78036fa8-a915-466e-aaef-c8d1f08c7bd7/input_2.fastq', 'phlat_index.tar.gz': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpy_dkrea9/78036fa8-a915-466e-aaef-c8d1f08c7bd7/phlat_index.tar.gz'} +<========= +Log from job 'run_bwa' kind-run_bwa/instance-fvcm11nr follows: +=========> + [2020-11-15T01:06:38-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-15T01:06:38-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/alignment/dna.py", line 142, in run_bwa + tool_version=bwa_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-15T01:20:32-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +Log from job 'run_phlat' kind-run_phlat/instance-z35qoqn5 follows: +=========> + [2020-11-15T01:35:02-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-15T01:35:02-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-15T01:50:47-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpf8x9yyk2/1454df9f-4b67-4586-b6d7-e59750b2b923/input_1.fastq', 'input_2.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpf8x9yyk2/1454df9f-4b67-4586-b6d7-e59750b2b923/input_2.fastq', 'phlat_index.tar.gz': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpf8x9yyk2/1454df9f-4b67-4586-b6d7-e59750b2b923/phlat_index.tar.gz'} +<========= +Log from job 'run_cutadapt' kind-run_cutadapt/instance-zdc0z58x follows: +=========> + [2020-11-15T01:36:32-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-15T01:36:32-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/qc/rna.py", line 59, in run_cutadapt + dockerhub=univ_options['dockerhub'], tool_version=cutadapt_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-15T01:40:09-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= diff --git a/errors/2020-11-17-10-01-21.txt b/errors/2020-11-17-10-01-21.txt new file mode 100644 index 00000000..35f783f3 --- /dev/null +++ b/errors/2020-11-17-10-01-21.txt @@ -0,0 +1,19 @@ +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 104, in initialize + os.mkdir(self.jobStoreDir) +FileExistsError: [Errno 17] File exists: '/scratch/drkthomp/jobStore' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/bin/ProTECT", line 11, in + load_entry_point('protect', 'console_scripts', 'ProTECT')() + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1025, in main + Job.Runner.startToil(start, params) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 794, in startToil + with Toil(options) as toil: + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 774, in __enter__ + jobStore.initialize(config) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 107, in initialize + raise JobStoreExistsException(self.jobStoreDir) +toil.jobStores.abstractJobStore.JobStoreExistsException: The job store '/scratch/drkthomp/jobStore' already exists. Use --restart to resume the workflow, or remove the job store with 'toil clean' to start the workflow from scratch. diff --git a/errors/2020-11-17-10-01-52.txt b/errors/2020-11-17-10-01-52.txt new file mode 100644 index 00000000..8365a537 --- /dev/null +++ b/errors/2020-11-17-10-01-52.txt @@ -0,0 +1,810 @@ +[2020-11-17T10:01:55-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_cutadapt/instance-zdc0z58x +[2020-11-17T10:01:55-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_bwa/instance-fvcm11nr +[2020-11-17T10:01:55-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-parse_config_file/instance-ml5szru_ +[2020-11-17T10:01:55-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-3chuyae2 +[2020-11-17T10:01:55-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-rssn99a0 +[2020-11-17T10:01:55-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-z35qoqn5 +[2020-11-17T10:01:55-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_bwa/instance-bou2djh0 +[2020-11-17T10:01:55-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-17T10:01:56-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-3chuyae2 with job batch system ID: 0 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-17T10:01:56-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-bou2djh0 with job batch system ID: 1 and cores: 80, disk: 13.4 G, and memory: 12.0 G +[2020-11-17T10:01:56-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-z35qoqn5 with job batch system ID: 2 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-17T10:01:56-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-fvcm11nr with job batch system ID: 3 and cores: 80, disk: 12.8 G, and memory: 12.0 G +[2020-11-17T10:01:56-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-rssn99a0 with job batch system ID: 4 and cores: 80, disk: 8.2 G, and memory: 2.0 G +[2020-11-17T10:01:56-0800] [MainThread] [I] [toil.leader] Issued job 'run_cutadapt' kind-run_cutadapt/instance-zdc0z58x with job batch system ID: 5 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T10:01:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmplcmx1t4b/worker_log.txt +[2020-11-17T10:01:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpny6uf6cr/worker_log.txt +[2020-11-17T10:01:58-0800] [MainThread] [I] [toil.leader] 2 jobs are running, 4 jobs are issued and waiting to run +[2020-11-17T10:18:23-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-3chuyae2. +[2020-11-17T10:18:23-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-3chuyae2 +[2020-11-17T10:18:23-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-3chuyae2 +[2020-11-17T10:18:23-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-3chuyae2 follows: +=========> + [2020-11-17T10:01:56-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T10:01:56-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 62, in run_phlat + input_files['phlat_index'] = untargz(input_files['phlat_index.tar.gz'], work_dir) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 158, in untargz + tarball.extractall(path=untar_to_dir) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/tarfile.py", line 2008, in extractall + numeric_owner=numeric_owner) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/tarfile.py", line 2050, in extract + numeric_owner=numeric_owner) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/tarfile.py", line 2120, in _extract_member + self.makefile(tarinfo, targetpath) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/tarfile.py", line 2169, in makefile + copyfileobj(source, target, tarinfo.size, ReadError, bufsize) + OSError: [Errno 122] Disk quota exceeded + [2020-11-17T10:18:19-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmplcmx1t4b/0a69aeb0-5ac3-420c-a5fd-5f1f1af93eec/input_1.fastq', 'input_2.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmplcmx1t4b/0a69aeb0-5ac3-420c-a5fd-5f1f1af93eec/input_2.fastq', 'phlat_index.tar.gz': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmplcmx1t4b/0a69aeb0-5ac3-420c-a5fd-5f1f1af93eec/phlat_index.tar.gz'} +<========= +[2020-11-17T10:18:23-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-3chuyae2 with ID kind-run_phlat/instance-3chuyae2 to 1 +[2020-11-17T10:18:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-3chuyae2 with job batch system ID: 6 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-17T10:18:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpa94_fc_n/worker_log.txt +[2020-11-17T10:23:38-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_bwa file:/scratch/drkthomp/jobStore kind-run_bwa/instance-bou2djh0. +[2020-11-17T10:23:38-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_bwa' kind-run_bwa/instance-bou2djh0 +[2020-11-17T10:23:38-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_bwa' kind-run_bwa/instance-bou2djh0 +[2020-11-17T10:23:38-0800] [MainThread] [W] [toil.leader] Log from job kind-run_bwa/instance-bou2djh0 follows: +=========> + [2020-11-17T10:01:56-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T10:01:56-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + 0.7.9a: Pulling from aarjunrao/bwa + 943c334059c7: Pulling fs layer + a1acf99303d2: Pulling fs layer + 27616aacb7b3: Pulling fs layer + 35d12cd1c9fc: Pulling fs layer + a3ed95caeb02: Pulling fs layer + 35d12cd1c9fc: Waiting + a3ed95caeb02: Waiting + a27d4604b3b3: Pulling fs layer + b23184322fd8: Pulling fs layer + a27d4604b3b3: Waiting + 45490e2c3ebf: Pulling fs layer + b23184322fd8: Waiting + 07bc45a5d3f9: Pulling fs layer + 2211351c9e85: Pulling fs layer + 45490e2c3ebf: Waiting + db8e7509afc0: Pulling fs layer + 2211351c9e85: Waiting + 27616aacb7b3: Download complete + a1acf99303d2: Download complete + a3ed95caeb02: Verifying Checksum + 35d12cd1c9fc: Download complete + 943c334059c7: Verifying Checksum + 943c334059c7: Download complete + b23184322fd8: Verifying Checksum + b23184322fd8: Download complete + 45490e2c3ebf: Download complete + 07bc45a5d3f9: Verifying Checksum + 07bc45a5d3f9: Download complete + 2211351c9e85: Verifying Checksum + 2211351c9e85: Download complete + db8e7509afc0: Verifying Checksum + db8e7509afc0: Download complete + 943c334059c7: Pull complete + a1acf99303d2: Pull complete + 27616aacb7b3: Pull complete + 35d12cd1c9fc: Pull complete + a3ed95caeb02: Pull complete + a27d4604b3b3: Verifying Checksum + a27d4604b3b3: Download complete + a27d4604b3b3: Pull complete + b23184322fd8: Pull complete + 45490e2c3ebf: Pull complete + 07bc45a5d3f9: Pull complete + 2211351c9e85: Pull complete + db8e7509afc0: Pull complete + Digest: sha256:dbd3bf232ec8e2371bb10bc108393bdb5c7ee184bdae783a53d165f36639245f + Status: Downloaded newer image for aarjunrao/bwa:0.7.9a + [M::mem_pestat] analyzing insert size distribution for orientation FF... + [M::mem_pestat] (25, 50, 75) percentile: (744, 1518, 3419) + [M::mem_pestat] low and high boundaries for computing mean and std.dev: (1, 8769) + [M::mem_pestat] mean and std.dev: (2133.63, 1924.92) + [M::mem_pestat] low and high boundaries for proper pairs: (1, 11444) + [M::mem_pestat] analyzing insert size distribution for orientation FR... + [M::mem_pestat] (25, 50, 75) percentile: (233, 257, 275) + [M::mem_pestat] low and high boundaries for computing mean and std.dev: (149, 359) + [M::mem_pestat] mean and std.dev: (254.01, 30.06) + [M::mem_pestat] low and high boundaries for proper pairs: (107, 401) + [M::mem_pestat] analyzing insert size distribution for orientation RF... + [M::mem_pestat] (25, 50, 75) percentile: (111, 125, 146) + [M::mem_pestat] low and high boundaries for computing mean and std.dev: (41, 216) + [M::mem_pestat] mean and std.dev: (127.10, 25.26) + [M::mem_pestat] low and high boundaries for proper pairs: (6, 251) + [M::mem_pestat] analyzing insert size distribution for orientation RR... + [M::mem_pestat] (25, 50, 75) percentile: (865, 1736, 3570) + [M::mem_pestat] low and high boundaries for computing mean and std.dev: (1, 8980) + [M::mem_pestat] mean and std.dev: (2498.64, 2101.67) + [M::mem_pestat] low and high boundaries for proper pairs: (1, 11685) + [M::mem_pestat] skip orientation FF + [M::mem_pestat] skip orientation RF + [M::mem_pestat] skip orientation RR + [main] Version: 0.7.9a-r786 + [main] CMD: /usr/local/bin/bwa mem -t 80.0 -v 1 /data/bwa_indexes/hg38 /data/dna_1.fastq /data/dna_2.fastq + [main] Real time: 110.481 sec; CPU: 2127.537 sec + chown: changing ownership of '/data/.jobState': Operation not permitted + chown: changing ownership of '/data/dna_1.fastq': Operation not permitted + chown: changing ownership of '/data/dna_2.fastq': Operation not permitted + chown: changing ownership of '/data/bwa_index.tar.gz': Operation not permitted + chown: changing ownership of '/data/bwa_indexes/hg38.amb': Operation not permitted + chown: changing ownership of '/data/bwa_indexes/hg38.sa': Operation not permitted + chown: changing ownership of '/data/bwa_indexes/hg38.bwt': Operation not permitted + chown: changing ownership of '/data/bwa_indexes/hg38.pac': Operation not permitted + chown: changing ownership of '/data/bwa_indexes/hg38.ann': Operation not permitted + chown: changing ownership of '/data/bwa_indexes': Operation not permitted + chown: changing ownership of '/data/tumor_dna.sam': Operation not permitted + chown: changing ownership of '/data': Operation not permitted + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpny6uf6cr/59673ac1-349e-46f0-9a45-69a2d56c4e51:/data', '--log-driver=none', 'aarjunrao/bwa:0.7.9a', 'mem', '-t', '80.0', '-v', '1', '/data/bwa_indexes/hg38', '/data/dna_1.fastq', '/data/dna_2.fastq']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/alignment/dna.py", line 142, in run_bwa + tool_version=bwa_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpny6uf6cr/59673ac1-349e-46f0-9a45-69a2d56c4e51:/data --log-driver=none aarjunrao/bwa:0.7.9a mem -t 80.0 -v 1 /data/bwa_indexes/hg38 /data/dna_1.fastq /data/dna_2.fastq" + [2020-11-17T10:23:35-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-17T10:23:38-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_bwa' kind-run_bwa/instance-bou2djh0 with ID kind-run_bwa/instance-bou2djh0 to 1 +[2020-11-17T10:23:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-bou2djh0 with job batch system ID: 7 and cores: 80, disk: 13.4 G, and memory: 12.0 G +[2020-11-17T10:23:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp_d21ezs7/worker_log.txt +[2020-11-17T10:32:06-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-z35qoqn5. +[2020-11-17T10:32:06-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-z35qoqn5 +[2020-11-17T10:32:06-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-z35qoqn5 +[2020-11-17T10:32:06-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-z35qoqn5 follows: +=========> + [2020-11-17T10:18:24-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T10:18:24-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + 1.0: Pulling from aarjunrao/phlat + 943c334059c7: Already exists + a1acf99303d2: Already exists + 27616aacb7b3: Already exists + 35d12cd1c9fc: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + a27d4604b3b3: Already exists + b23184322fd8: Already exists + 45490e2c3ebf: Already exists + a3ed95caeb02: Already exists + 07bc45a5d3f9: Already exists + a3ed95caeb02: Already exists + 4e8282f9b0b2: Pulling fs layer + d8467ea2e124: Pulling fs layer + b53fc7d0498f: Pulling fs layer + 30ab0013e0b7: Pulling fs layer + 949d3b3c935b: Pulling fs layer + a98c227726f6: Pulling fs layer + 4347594c2d1a: Pulling fs layer + f13886bcc594: Pulling fs layer + 30ab0013e0b7: Waiting + 4347594c2d1a: Waiting + a3ed95caeb02: Pulling fs layer + 949d3b3c935b: Waiting + f13886bcc594: Waiting + a3ed95caeb02: Waiting + 4e8282f9b0b2: Verifying Checksum + 4e8282f9b0b2: Download complete + b53fc7d0498f: Download complete + 4e8282f9b0b2: Pull complete + d8467ea2e124: Verifying Checksum + d8467ea2e124: Download complete + d8467ea2e124: Pull complete + a98c227726f6: Download complete + b53fc7d0498f: Pull complete + 30ab0013e0b7: Verifying Checksum + 30ab0013e0b7: Download complete + 30ab0013e0b7: Pull complete + f13886bcc594: Download complete + 949d3b3c935b: Verifying Checksum + 949d3b3c935b: Download complete + 4347594c2d1a: Verifying Checksum + 4347594c2d1a: Download complete + a3ed95caeb02: Download complete + 949d3b3c935b: Pull complete + a98c227726f6: Pull complete + 4347594c2d1a: Pull complete + f13886bcc594: Pull complete + a3ed95caeb02: Pull complete + Digest: sha256:a11b92ed62d9e189ae36823abeb405c8bca9f41f47fd0d956397c539f8534b5f + Status: Downloaded newer image for aarjunrao/phlat:1.0 + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 59, in + nthreads=int(val) + ValueError: invalid literal for int() with base 10: '80.0' + chown: changing ownership of '/data/.jobState': Operation not permitted + chown: changing ownership of '/data/input_1.fastq': Operation not permitted + chown: changing ownership of '/data/input_2.fastq': Operation not permitted + chown: changing ownership of '/data/phlat_index.tar.gz': Operation not permitted + chown: changing ownership of '/data/index4phlat/ucsc.artHLA.rev.2.bt2': Operation not permitted + chown: changing ownership of '/data/index4phlat/ucsc.artHLA.4.bt2': Operation not permitted + chown: changing ownership of '/data/index4phlat/ucsc.artHLA.3.bt2': Operation not permitted + chown: changing ownership of '/data/index4phlat/ucsc.artHLA.rev.1.bt2': Operation not permitted + chown: changing ownership of '/data/index4phlat/ucsc.artHLA.2.bt2': Operation not permitted + chown: changing ownership of '/data/index4phlat/ucsc.artHLA.1.bt2': Operation not permitted + chown: changing ownership of '/data/index4phlat': Operation not permitted + chown: changing ownership of '/data': Operation not permitted + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpa94_fc_n/931d52d0-cfb2-41ae-a58a-17ac7019b925:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'normal_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '80.0']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpa94_fc_n/931d52d0-cfb2-41ae-a58a-17ac7019b925:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag normal_dna -e /home/phlat-1.0 -o /data -p 80.0" + [2020-11-17T10:32:02-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpa94_fc_n/931d52d0-cfb2-41ae-a58a-17ac7019b925/input_1.fastq', 'input_2.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpa94_fc_n/931d52d0-cfb2-41ae-a58a-17ac7019b925/input_2.fastq', 'phlat_index.tar.gz': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpa94_fc_n/931d52d0-cfb2-41ae-a58a-17ac7019b925/phlat_index.tar.gz'} +<========= +[2020-11-17T10:32:06-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-z35qoqn5 with ID kind-run_phlat/instance-z35qoqn5 to 1 +[2020-11-17T10:32:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-z35qoqn5 with job batch system ID: 8 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-17T10:32:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp8_w41v8t/worker_log.txt +[2020-11-17T10:35:15-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-rssn99a0. +[2020-11-17T10:35:15-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-rssn99a0 +[2020-11-17T10:35:15-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-rssn99a0 +[2020-11-17T10:35:15-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-rssn99a0 follows: +=========> + [2020-11-17T10:23:39-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T10:23:39-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 59, in + nthreads=int(val) + ValueError: invalid literal for int() with base 10: '80.0' + chown: changing ownership of '/data/.jobState': Operation not permitted + chown: changing ownership of '/data/input_1.fastq': Operation not permitted + chown: changing ownership of '/data/input_2.fastq': Operation not permitted + chown: changing ownership of '/data/phlat_index.tar.gz': Operation not permitted + chown: changing ownership of '/data/index4phlat/ucsc.artHLA.rev.2.bt2': Operation not permitted + chown: changing ownership of '/data/index4phlat/ucsc.artHLA.4.bt2': Operation not permitted + chown: changing ownership of '/data/index4phlat/ucsc.artHLA.3.bt2': Operation not permitted + chown: changing ownership of '/data/index4phlat/ucsc.artHLA.rev.1.bt2': Operation not permitted + chown: changing ownership of '/data/index4phlat/ucsc.artHLA.2.bt2': Operation not permitted + chown: changing ownership of '/data/index4phlat/ucsc.artHLA.1.bt2': Operation not permitted + chown: changing ownership of '/data/index4phlat': Operation not permitted + chown: changing ownership of '/data': Operation not permitted + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp_d21ezs7/ff0f1f0e-a853-4357-9246-fb5aa3feb95c:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_rna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '80.0']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp_d21ezs7/ff0f1f0e-a853-4357-9246-fb5aa3feb95c:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_rna -e /home/phlat-1.0 -o /data -p 80.0" + [2020-11-17T10:35:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp_d21ezs7/ff0f1f0e-a853-4357-9246-fb5aa3feb95c/input_1.fastq', 'input_2.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp_d21ezs7/ff0f1f0e-a853-4357-9246-fb5aa3feb95c/input_2.fastq', 'phlat_index.tar.gz': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp_d21ezs7/ff0f1f0e-a853-4357-9246-fb5aa3feb95c/phlat_index.tar.gz'} +<========= +[2020-11-17T10:35:15-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-rssn99a0 with ID kind-run_phlat/instance-rssn99a0 to 1 +[2020-11-17T10:35:15-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-rssn99a0 with job batch system ID: 9 and cores: 80, disk: 8.2 G, and memory: 2.0 G +[2020-11-17T10:35:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpn_evxpvz/worker_log.txt +[2020-11-17T10:35:16-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_cutadapt file:/scratch/drkthomp/jobStore kind-run_cutadapt/instance-zdc0z58x. +[2020-11-17T10:35:16-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_cutadapt' kind-run_cutadapt/instance-zdc0z58x +[2020-11-17T10:35:16-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_cutadapt' kind-run_cutadapt/instance-zdc0z58x +[2020-11-17T10:35:16-0800] [MainThread] [W] [toil.leader] Log from job kind-run_cutadapt/instance-zdc0z58x follows: +=========> + [2020-11-17T10:32:48-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T10:32:48-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + 1.9.1: Pulling from aarjunrao/cutadapt + 943c334059c7: Already exists + a1acf99303d2: Already exists + 27616aacb7b3: Already exists + 35d12cd1c9fc: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + a27d4604b3b3: Already exists + b23184322fd8: Already exists + 45490e2c3ebf: Already exists + a3ed95caeb02: Already exists + 07bc45a5d3f9: Already exists + a3ed95caeb02: Already exists + e99a83101e51: Pulling fs layer + c948cbf08432: Pulling fs layer + 6554961e8521: Pulling fs layer + a3ed95caeb02: Pulling fs layer + a3ed95caeb02: Waiting + 6554961e8521: Verifying Checksum + 6554961e8521: Download complete + a3ed95caeb02: Download complete + c948cbf08432: Verifying Checksum + c948cbf08432: Download complete + e99a83101e51: Verifying Checksum + e99a83101e51: Download complete + e99a83101e51: Pull complete + c948cbf08432: Pull complete + 6554961e8521: Pull complete + a3ed95caeb02: Pull complete + Digest: sha256:0bb80f77facdceb34bb619c871760b2035391a0c17ef35873898f03a18eb49e2 + Status: Downloaded newer image for aarjunrao/cutadapt:1.9.1 + This is cutadapt 1.9.1 with Python 2.7.6 + Command line parameters: -a AGATCGGAAGAG -A AGATCGGAAGAG -m 35 -o /data/rna_cutadapt_1.fastq.gz -p /data/rna_cutadapt_2.fastq.gz /data/rna_1.fastq /data/rna_2.fastq + Trimming 2 adapters with at most 10.0% errors in paired-end mode ... + Finished in 45.30 s (29 us/read; 2.07 M reads/minute). + + === Summary === + + Total read pairs processed: 1,562,357 + Read 1 with adapter: 46,459 (3.0%) + Read 2 with adapter: 45,067 (2.9%) + Pairs that were too short: 2,154 (0.1%) + Pairs written (passing filters): 1,560,203 (99.9%) + + Total basepairs processed: 312,471,400 bp + Read 1: 156,235,700 bp + Read 2: 156,235,700 bp + Total written (filtered): 311,600,470 bp (99.7%) + Read 1: 155,800,190 bp + Read 2: 155,800,280 bp + + === First read: Adapter 1 === + + Sequence: AGATCGGAAGAG; Type: regular 3'; Length: 12; Trimmed: 46459 times. + + No. of allowed errors: + 0-9 bp: 0; 10-12 bp: 1 + + Bases preceding removed adapters: + A: 32.1% + C: 27.2% + G: 23.0% + T: 13.9% + none/other: 3.8% + + Overview of removed sequences + length count expect max.err error counts + 3 31191 24411.8 0 31191 + 4 6994 6103.0 0 6994 + 5 1726 1525.7 0 1726 + 6 408 381.4 0 408 + 7 294 95.4 0 294 + 8 224 23.8 0 224 + 9 244 6.0 0 220 24 + 10 235 1.5 1 196 39 + 11 263 0.4 1 223 40 + 12 169 0.1 1 161 8 + 13 177 0.1 1 170 7 + 14 160 0.1 1 153 7 + 15 148 0.1 1 144 4 + 16 154 0.1 1 146 8 + 17 127 0.1 1 118 9 + 18 109 0.1 1 101 8 + 19 108 0.1 1 101 7 + 20 99 0.1 1 92 7 + 21 107 0.1 1 98 9 + 22 84 0.1 1 78 6 + 23 87 0.1 1 81 6 + 24 74 0.1 1 66 8 + 25 74 0.1 1 66 8 + 26 70 0.1 1 65 5 + 27 68 0.1 1 62 6 + 28 75 0.1 1 68 7 + 29 60 0.1 1 56 4 + 30 52 0.1 1 44 8 + 31 53 0.1 1 50 3 + 32 47 0.1 1 39 8 + 33 69 0.1 1 63 6 + 34 53 0.1 1 48 5 + 35 67 0.1 1 62 5 + 36 46 0.1 1 39 7 + 37 29 0.1 1 26 3 + 38 46 0.1 1 41 5 + 39 31 0.1 1 29 2 + 40 31 0.1 1 24 7 + 41 37 0.1 1 31 6 + 42 36 0.1 1 28 8 + 43 23 0.1 1 18 5 + 44 26 0.1 1 18 8 + 45 31 0.1 1 23 8 + 46 24 0.1 1 14 10 + 47 25 0.1 1 21 4 + 48 17 0.1 1 14 3 + 49 23 0.1 1 18 5 + 50 15 0.1 1 12 3 + 51 23 0.1 1 19 4 + 52 12 0.1 1 9 3 + 53 17 0.1 1 12 5 + 54 17 0.1 1 11 6 + 55 11 0.1 1 9 2 + 56 12 0.1 1 5 7 + 57 10 0.1 1 6 4 + 58 8 0.1 1 3 5 + 59 11 0.1 1 7 4 + 60 8 0.1 1 2 6 + 61 9 0.1 1 6 3 + 62 6 0.1 1 3 3 + 63 9 0.1 1 4 5 + 64 12 0.1 1 6 6 + 65 11 0.1 1 5 6 + 66 9 0.1 1 3 6 + 67 7 0.1 1 0 7 + 68 9 0.1 1 2 7 + 69 7 0.1 1 3 4 + 70 10 0.1 1 3 7 + 71 6 0.1 1 0 6 + 72 7 0.1 1 1 6 + 73 5 0.1 1 0 5 + 74 7 0.1 1 2 5 + 75 6 0.1 1 1 5 + 76 2 0.1 1 0 2 + 77 5 0.1 1 0 5 + 78 7 0.1 1 4 3 + 79 5 0.1 1 0 5 + 80 8 0.1 1 2 6 + 81 1 0.1 1 1 + 82 4 0.1 1 1 3 + 83 6 0.1 1 1 5 + 84 4 0.1 1 1 3 + 85 4 0.1 1 1 3 + 86 6 0.1 1 2 4 + 87 2 0.1 1 0 2 + 88 6 0.1 1 0 6 + 89 9 0.1 1 1 8 + 90 6 0.1 1 1 5 + 91 5 0.1 1 0 5 + 92 6 0.1 1 2 4 + 93 5 0.1 1 0 5 + 94 4 0.1 1 0 4 + 95 7 0.1 1 0 7 + 96 4 0.1 1 0 4 + 97 4 0.1 1 0 4 + 98 2 0.1 1 0 2 + 99 5 0.1 1 0 5 + 100 1783 0.1 1 0 1783 + + === Second read: Adapter 2 === + + Sequence: AGATCGGAAGAG; Type: regular 3'; Length: 12; Trimmed: 45067 times. + + No. of allowed errors: + 0-9 bp: 0; 10-12 bp: 1 + + Bases preceding removed adapters: + A: 32.9% + C: 27.6% + G: 23.8% + T: 14.4% + none/other: 1.4% + + Overview of removed sequences + length count expect max.err error counts + 3 31015 24411.8 0 31015 + 4 6968 6103.0 0 6968 + 5 1707 1525.7 0 1707 + 6 392 381.4 0 392 + 7 290 95.4 0 290 + 8 223 23.8 0 223 + 9 236 6.0 0 211 25 + 10 220 1.5 1 195 25 + 11 250 0.4 1 215 35 + 12 176 0.1 1 155 21 + 13 176 0.1 1 168 8 + 14 157 0.1 1 148 9 + 15 155 0.1 1 143 12 + 16 155 0.1 1 145 10 + 17 129 0.1 1 116 13 + 18 109 0.1 1 98 11 + 19 109 0.1 1 98 11 + 20 99 0.1 1 87 12 + 21 109 0.1 1 101 8 + 22 82 0.1 1 74 8 + 23 88 0.1 1 76 12 + 24 71 0.1 1 60 11 + 25 75 0.1 1 65 10 + 26 71 0.1 1 63 8 + 27 63 0.1 1 58 5 + 28 74 0.1 1 70 4 + 29 58 0.1 1 52 6 + 30 51 0.1 1 45 6 + 31 58 0.1 1 49 9 + 32 44 0.1 1 37 7 + 33 71 0.1 1 61 10 + 34 54 0.1 1 45 9 + 35 69 0.1 1 64 5 + 36 45 0.1 1 39 6 + 37 29 0.1 1 24 5 + 38 47 0.1 1 36 11 + 39 35 0.1 1 26 9 + 40 34 0.1 1 25 9 + 41 39 0.1 1 32 7 + 42 33 0.1 1 28 5 + 43 26 0.1 1 18 8 + 44 29 0.1 1 16 13 + 45 28 0.1 1 22 6 + 46 16 0.1 1 13 3 + 47 23 0.1 1 19 4 + 48 20 0.1 1 12 8 + 49 27 0.1 1 19 8 + 50 21 0.1 1 14 7 + 51 30 0.1 1 20 10 + 52 12 0.1 1 8 4 + 53 15 0.1 1 10 5 + 54 16 0.1 1 11 5 + 55 11 0.1 1 10 1 + 56 13 0.1 1 6 7 + 57 13 0.1 1 6 7 + 58 12 0.1 1 4 8 + 59 12 0.1 1 6 6 + 60 6 0.1 1 1 5 + 61 8 0.1 1 4 4 + 62 6 0.1 1 2 4 + 63 10 0.1 1 5 5 + 64 9 0.1 1 5 4 + 65 13 0.1 1 5 8 + 66 13 0.1 1 2 11 + 67 6 0.1 1 2 4 + 68 5 0.1 1 2 3 + 69 6 0.1 1 3 3 + 70 6 0.1 1 3 3 + 71 3 0.1 1 0 3 + 72 6 0.1 1 2 4 + 73 11 0.1 1 2 9 + 74 15 0.1 1 2 13 + 75 7 0.1 1 0 7 + 76 7 0.1 1 0 7 + 77 5 0.1 1 0 5 + 78 9 0.1 1 5 4 + 79 7 0.1 1 2 5 + 80 6 0.1 1 2 4 + 81 5 0.1 1 1 4 + 82 6 0.1 1 2 4 + 83 7 0.1 1 1 6 + 84 4 0.1 1 0 4 + 85 1 0.1 1 1 + 86 5 0.1 1 1 4 + 87 12 0.1 1 1 11 + 88 3 0.1 1 0 3 + 89 5 0.1 1 2 3 + 90 3 0.1 1 1 2 + 91 8 0.1 1 1 7 + 92 6 0.1 1 1 5 + 93 2 0.1 1 0 2 + 94 4 0.1 1 0 4 + 95 3 0.1 1 0 3 + 96 3 0.1 1 0 3 + 97 1 0.1 1 0 1 + 98 4 0.1 1 0 4 + 99 6 0.1 1 0 6 + 100 625 0.1 1 0 625 + + chown: changing ownership of '/data/.jobState': Operation not permitted + chown: changing ownership of '/data/rna_1.fastq': Operation not permitted + chown: changing ownership of '/data/rna_2.fastq': Operation not permitted + chown: changing ownership of '/data/rna_cutadapt_1.fastq.gz': Operation not permitted + chown: changing ownership of '/data/rna_cutadapt_2.fastq.gz': Operation not permitted + chown: changing ownership of '/data': Operation not permitted + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp8_w41v8t/fed56e00-bdad-4d5b-b416-0e7a320a3433:/data', '--log-driver=none', 'aarjunrao/cutadapt:1.9.1', '-a', 'AGATCGGAAGAG', '-A', 'AGATCGGAAGAG', '-m', '35', '-o', '/data/rna_cutadapt_1.fastq.gz', '-p', '/data/rna_cutadapt_2.fastq.gz', '/data/rna_1.fastq', '/data/rna_2.fastq']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/qc/rna.py", line 59, in run_cutadapt + dockerhub=univ_options['dockerhub'], tool_version=cutadapt_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp8_w41v8t/fed56e00-bdad-4d5b-b416-0e7a320a3433:/data --log-driver=none aarjunrao/cutadapt:1.9.1 -a AGATCGGAAGAG -A AGATCGGAAGAG -m 35 -o /data/rna_cutadapt_1.fastq.gz -p /data/rna_cutadapt_2.fastq.gz /data/rna_1.fastq /data/rna_2.fastq" + [2020-11-17T10:35:16-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-17T10:35:16-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_cutadapt' kind-run_cutadapt/instance-zdc0z58x with ID kind-run_cutadapt/instance-zdc0z58x to 1 +[2020-11-17T10:35:16-0800] [MainThread] [I] [toil.leader] Issued job 'run_cutadapt' kind-run_cutadapt/instance-zdc0z58x with job batch system ID: 10 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T10:35:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpvxfdc39j/worker_log.txt +[2020-11-17T10:49:11-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_bwa file:/scratch/drkthomp/jobStore kind-run_bwa/instance-fvcm11nr. +[2020-11-17T10:49:11-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_bwa' kind-run_bwa/instance-fvcm11nr +[2020-11-17T10:49:11-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_bwa' kind-run_bwa/instance-fvcm11nr +[2020-11-17T10:49:11-0800] [MainThread] [W] [toil.leader] Log from job kind-run_bwa/instance-fvcm11nr follows: +=========> + [2020-11-17T10:35:16-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T10:35:16-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/alignment/dna.py", line 130, in run_bwa + input_files['bwa_index'] = untargz(input_files['bwa_index.tar.gz'], work_dir) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 158, in untargz + tarball.extractall(path=untar_to_dir) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/tarfile.py", line 2008, in extractall + numeric_owner=numeric_owner) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/tarfile.py", line 2050, in extract + numeric_owner=numeric_owner) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/tarfile.py", line 2120, in _extract_member + self.makefile(tarinfo, targetpath) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/tarfile.py", line 2161, in makefile + with bltn_open(targetpath, "wb") as target: + OSError: [Errno 122] Disk quota exceeded: '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpn_evxpvz/f8e26626-c5d8-4a3d-b307-d390674cd8c3/bwa_indexes/hg38.ann' + [2020-11-17T10:49:07-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-17T10:49:11-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_bwa' kind-run_bwa/instance-fvcm11nr with ID kind-run_bwa/instance-fvcm11nr to 1 +[2020-11-17T10:49:11-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-fvcm11nr with job batch system ID: 11 and cores: 80, disk: 12.8 G, and memory: 12.0 G +[2020-11-17T10:49:12-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpqui9rmov/worker_log.txt +[2020-11-17T10:49:31-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-rssn99a0. +[2020-11-17T10:49:31-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-rssn99a0 +[2020-11-17T10:49:31-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-rssn99a0 +[2020-11-17T10:49:31-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-rssn99a0 follows: +=========> + [2020-11-17T10:35:32-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T10:35:32-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 62, in run_phlat + input_files['phlat_index'] = untargz(input_files['phlat_index.tar.gz'], work_dir) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 158, in untargz + tarball.extractall(path=untar_to_dir) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/tarfile.py", line 2008, in extractall + numeric_owner=numeric_owner) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/tarfile.py", line 2050, in extract + numeric_owner=numeric_owner) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/tarfile.py", line 2120, in _extract_member + self.makefile(tarinfo, targetpath) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/tarfile.py", line 2169, in makefile + copyfileobj(source, target, tarinfo.size, ReadError, bufsize) + OSError: [Errno 122] Disk quota exceeded + [2020-11-17T10:49:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpvxfdc39j/040194b9-67ca-4865-b990-915887dbbaea/input_1.fastq', 'input_2.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpvxfdc39j/040194b9-67ca-4865-b990-915887dbbaea/input_2.fastq', 'phlat_index.tar.gz': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpvxfdc39j/040194b9-67ca-4865-b990-915887dbbaea/phlat_index.tar.gz'} +<========= +[2020-11-17T10:49:31-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-rssn99a0 with ID kind-run_phlat/instance-rssn99a0 to 0 +[2020-11-17T10:49:31-0800] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-rssn99a0 with ID kind-run_phlat/instance-rssn99a0 is completely failed +[2020-11-17T10:49:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpkw799set/worker_log.txt +[2020-11-17T11:01:59-0800] [MainThread] [I] [toil.leader] 2 jobs are running, 3 jobs are issued and waiting to run +[2020-11-17T11:05:00-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-z35qoqn5. +[2020-11-17T11:05:00-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-z35qoqn5 +[2020-11-17T11:05:00-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-z35qoqn5 +[2020-11-17T11:05:00-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-z35qoqn5 follows: +=========> + [2020-11-17T10:49:12-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T10:49:12-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 59, in + nthreads=int(val) + ValueError: invalid literal for int() with base 10: '80.0' + chown: changing ownership of '/data/.jobState': Operation not permitted + chown: changing ownership of '/data/input_1.fastq': Operation not permitted + chown: changing ownership of '/data/input_2.fastq': Operation not permitted + chown: changing ownership of '/data/phlat_index.tar.gz': Operation not permitted + chown: changing ownership of '/data/index4phlat/ucsc.artHLA.rev.2.bt2': Operation not permitted + chown: changing ownership of '/data/index4phlat/ucsc.artHLA.4.bt2': Operation not permitted + chown: changing ownership of '/data/index4phlat/ucsc.artHLA.3.bt2': Operation not permitted + chown: changing ownership of '/data/index4phlat/ucsc.artHLA.rev.1.bt2': Operation not permitted + chown: changing ownership of '/data/index4phlat/ucsc.artHLA.2.bt2': Operation not permitted + chown: changing ownership of '/data/index4phlat/ucsc.artHLA.1.bt2': Operation not permitted + chown: changing ownership of '/data/index4phlat': Operation not permitted + chown: changing ownership of '/data': Operation not permitted + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpqui9rmov/2cab0a37-e05e-4fc3-804c-a5e699d961ee:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'normal_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '80.0']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpqui9rmov/2cab0a37-e05e-4fc3-804c-a5e699d961ee:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag normal_dna -e /home/phlat-1.0 -o /data -p 80.0" + [2020-11-17T11:04:57-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpqui9rmov/2cab0a37-e05e-4fc3-804c-a5e699d961ee/input_1.fastq', 'input_2.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpqui9rmov/2cab0a37-e05e-4fc3-804c-a5e699d961ee/input_2.fastq', 'phlat_index.tar.gz': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpqui9rmov/2cab0a37-e05e-4fc3-804c-a5e699d961ee/phlat_index.tar.gz'} +<========= +[2020-11-17T11:05:00-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-z35qoqn5 with ID kind-run_phlat/instance-z35qoqn5 to 0 +[2020-11-17T11:05:00-0800] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-z35qoqn5 with ID kind-run_phlat/instance-z35qoqn5 is completely failed +[2020-11-17T11:05:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpjt457_86/worker_log.txt +[2020-11-17T11:05:17-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-3chuyae2. +[2020-11-17T11:05:17-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-3chuyae2 +[2020-11-17T11:05:17-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-3chuyae2 +[2020-11-17T11:05:17-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-3chuyae2 follows: +=========> + [2020-11-17T10:49:32-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T10:49:32-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 62, in run_phlat + input_files['phlat_index'] = untargz(input_files['phlat_index.tar.gz'], work_dir) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 158, in untargz + tarball.extractall(path=untar_to_dir) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/tarfile.py", line 2008, in extractall + numeric_owner=numeric_owner) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/tarfile.py", line 2050, in extract + numeric_owner=numeric_owner) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/tarfile.py", line 2120, in _extract_member + self.makefile(tarinfo, targetpath) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/tarfile.py", line 2169, in makefile + copyfileobj(source, target, tarinfo.size, ReadError, bufsize) + OSError: [Errno 122] Disk quota exceeded + [2020-11-17T11:05:00-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpkw799set/3decbccd-c3c7-428b-90a8-f6bb79745ba2/input_1.fastq', 'input_2.fastq': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpkw799set/3decbccd-c3c7-428b-90a8-f6bb79745ba2/input_2.fastq', 'phlat_index.tar.gz': '/private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmpkw799set/3decbccd-c3c7-428b-90a8-f6bb79745ba2/phlat_index.tar.gz'} +<========= +[2020-11-17T11:05:17-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-3chuyae2 with ID kind-run_phlat/instance-3chuyae2 to 0 +[2020-11-17T11:05:17-0800] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-3chuyae2 with ID kind-run_phlat/instance-3chuyae2 is completely failed +[2020-11-17T11:05:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /private/home/drkthomp/protect3/workDir/node-60141861-48d9-4a88-81d8-ba092a076b7d-e8611b53-bdab-4598-acae-87977f99831e/tmp5gyyr1e8/worker_log.txt diff --git a/errors/2020-11-17-11-07-31.txt b/errors/2020-11-17-11-07-31.txt new file mode 100644 index 00000000..bf05bbe0 --- /dev/null +++ b/errors/2020-11-17-11-07-31.txt @@ -0,0 +1,12 @@ +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/bin/ProTECT", line 11, in + load_entry_point('protect', 'console_scripts', 'ProTECT')() + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1025, in main + Job.Runner.startToil(start, params) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 794, in startToil + with Toil(options) as toil: + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 770, in __enter__ + config.setOptions(self.options) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 202, in setOptions + % self.workDir) +RuntimeError: The path provided to --workDir (/scratch/drkthomp/workDir) does not exist. diff --git a/errors/2020-11-17-11-08-36.txt b/errors/2020-11-17-11-08-36.txt new file mode 100644 index 00000000..2ad61bab --- /dev/null +++ b/errors/2020-11-17-11-08-36.txt @@ -0,0 +1,3537 @@ +[2020-11-17T11:08:37-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-17T11:08:37-0800] [MainThread] [I] [toil.leader] Issued job 'parse_config_file' kind-parse_config_file/instance-rsybzib_ with job batch system ID: 0 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_e1i6qut/worker_log.txt +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Job ended: 'parse_config_file' kind-parse_config_file/instance-rsybzib_ +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] 0 jobs are running, 0 jobs are issued and waiting to run +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-xwqqulqx with job batch system ID: 1 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-pm0rq1eo with job batch system ID: 2 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-8rr6t8z5 with job batch system ID: 3 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-d8bsw41o with job batch system ID: 4 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-rk6h0_mb with job batch system ID: 5 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-413evf10 with job batch system ID: 6 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-uw_beg1k with job batch system ID: 7 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-avivvmbu with job batch system ID: 8 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-wzjf3rph with job batch system ID: 9 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-e6b8bx5m with job batch system ID: 10 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1n5n04pl with job batch system ID: 11 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ri5xqawd with job batch system ID: 12 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-0lt3sox1 with job batch system ID: 13 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-zpiml3kc with job batch system ID: 14 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-efiej08t with job batch system ID: 15 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-n2gfle63 with job batch system ID: 16 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-c1ayvcxz with job batch system ID: 17 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-pnnl7eoi with job batch system ID: 18 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-hwylwa35 with job batch system ID: 19 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-3d1cxz6_ with job batch system ID: 20 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-3_8akmbt with job batch system ID: 21 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-d74j5wjn with job batch system ID: 22 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-7fnoe8ok with job batch system ID: 23 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-wp8lltdy with job batch system ID: 24 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5m61ti75 with job batch system ID: 25 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-n2yosdx7 with job batch system ID: 26 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-gyc__eda with job batch system ID: 27 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-i7dxi6sl with job batch system ID: 28 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-9xv01ni5 with job batch system ID: 29 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:08:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:39: Parsing config file +[2020-11-17T11:08:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:39: Obtaining tool inputs +[2020-11-17T11:08:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:39: Obtained tool inputs +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp2_t9hisa/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmprou4mcxd/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpislvkeum/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp7e38jdgu/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpta0n9hsd/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmplo6nu2ij/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp7vn734uv/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpps4ezfnw/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpap1oow3_/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpa7xtzpga/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmplktu760b/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3f5zosus/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp20ljovnx/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_gko07y5/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpg8rt5414/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpew11n8io/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp1stpd05z/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpecwzerco/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp1vizali7/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpnvlxmkae/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpkv5emspq/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmppvjzscr5/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3hun5hds/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphesdw6zc/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpaqde9q50/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpmhijm8bn/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4wyt8e3h/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvy70r9g6/worker_log.txt +[2020-11-17T11:08:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmph7136cle/worker_log.txt +[2020-11-17T11:08:41-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-413evf10 +[2020-11-17T11:08:41-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-rk6h0_mb +[2020-11-17T11:08:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:41: Obtaining file (indexes:genome_dict) to the file job store +[2020-11-17T11:08:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:41: Obtaining file (indexes:genome_fai) to the file job store +[2020-11-17T11:08:42-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-zpiml3kc +[2020-11-17T11:08:42-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-n2yosdx7 +[2020-11-17T11:08:42-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-gyc__eda +[2020-11-17T11:08:42-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-efiej08t +[2020-11-17T11:08:42-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5m61ti75 +[2020-11-17T11:08:42-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-i7dxi6sl +[2020-11-17T11:08:42-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-c1ayvcxz +[2020-11-17T11:08:42-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-wp8lltdy +[2020-11-17T11:08:42-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-9xv01ni5 +[2020-11-17T11:08:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:42: Obtaining file (radia:retrogene_beds) to the file job store +[2020-11-17T11:08:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:42: Obtaining file (reports:mhc_pathways_file) to the file job store +[2020-11-17T11:08:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:42: Obtaining file (mhcii:method_file) to the file job store +[2020-11-17T11:08:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:42: Obtaining file (radia:pseudogene_beds) to the file job store +[2020-11-17T11:08:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:42: Obtaining file (reports:itx_resistance_file) to the file job store +[2020-11-17T11:08:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:42: Obtaining file (mhci:method_file) to the file job store +[2020-11-17T11:08:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:42: Obtaining file (reports:immune_resistance_pathways_file) to the file job store +[2020-11-17T11:08:42-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1n5n04pl +[2020-11-17T11:08:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:42: Obtaining file (strelka:config_file) to the file job store +[2020-11-17T11:08:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:42: Obtaining file (reports:car_t_targets_file) to the file job store +[2020-11-17T11:08:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:42: Obtaining file (indexes:dbsnp_tbi) to the file job store +[2020-11-17T11:08:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:42: Obtaining file (indexes:cosmic_idx) to the file job store +[2020-11-17T11:08:42-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-avivvmbu +[2020-11-17T11:08:42-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-n2gfle63 +[2020-11-17T11:08:42-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-e6b8bx5m +[2020-11-17T11:08:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:42: Obtaining file (radia:gencode_beds) to the file job store +[2020-11-17T11:08:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:42: Obtaining file (indexes:dbsnp_idx) to the file job store +[2020-11-17T11:08:43-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-hwylwa35 +[2020-11-17T11:08:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:43: Obtaining file (transgene:gencode_peptide_fasta) to the file job store +[2020-11-17T11:08:43-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-uw_beg1k +[2020-11-17T11:08:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:43: Obtaining file (indexes:cosmic_vcf) to the file job store +[2020-11-17T11:08:44-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:44: Obtaining file (radia:cosmic_beds) to the file job store +[2020-11-17T11:08:44-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ri5xqawd +[2020-11-17T11:08:44-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-3d1cxz6_ +[2020-11-17T11:08:44-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:44: Obtaining file (transgene:gencode_transcript_fasta) to the file job store +[2020-11-17T11:08:45-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-3_8akmbt +[2020-11-17T11:08:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:08:45: Obtaining file (transgene:gencode_annotation_gtf) to the file job store +[2020-11-17T11:09:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:09:16: Obtaining file (transgene:genome_fasta) to the file job store +[2020-11-17T11:09:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:09:16: Obtaining file (indexes:genome_fasta) to the file job store +[2020-11-17T11:09:16-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-d74j5wjn +[2020-11-17T11:09:16-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-d8bsw41o +[2020-11-17T11:09:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:09:16: Obtaining file (radia:dbsnp_beds) to the file job store +[2020-11-17T11:09:16-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-0lt3sox1 +[2020-11-17T11:09:20-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-pnnl7eoi +[2020-11-17T11:09:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:09:20: Obtaining file (snpeff:index) to the file job store +[2020-11-17T11:09:21-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-8rr6t8z5 +[2020-11-17T11:09:21-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:09:21: Obtaining file (rsem:index) to the file job store +[2020-11-17T11:09:56-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-wzjf3rph +[2020-11-17T11:09:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:09:56: Obtaining file (indexes:dbsnp_vcf) to the file job store +[2020-11-17T11:10:27-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:10:27: Obtaining file (bwa:index) to the file job store +[2020-11-17T11:10:27-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-pm0rq1eo +[2020-11-17T11:10:36-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-7fnoe8ok +[2020-11-17T11:10:36-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:10:36: Obtaining file (phlat:index) to the file job store +[2020-11-17T11:12:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:12:43: Obtaining file (star:index) to the file job store +[2020-11-17T11:12:44-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-xwqqulqx +[2020-11-17T11:12:44-0800] [MainThread] [I] [toil.leader] Issued job 'launch_protect' kind-launch_protect/instance-3gma8ru0 with job batch system ID: 30 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:12:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpz3s1k31j/worker_log.txt +[2020-11-17T11:12:46-0800] [MainThread] [I] [toil.leader] Job ended: 'launch_protect' kind-launch_protect/instance-3gma8ru0 +[2020-11-17T11:12:46-0800] [MainThread] [I] [toil.leader] Issued job 'prepare_samples' kind-prepare_samples/instance-kuhmgrea with job batch system ID: 31 and cores: 1, disk: 40.0 G, and memory: 2.0 G +[2020-11-17T11:12:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp00vkzsc0/worker_log.txt +[2020-11-17T11:12:58-0800] [MainThread] [I] [toil.leader] Job ended: 'prepare_samples' kind-prepare_samples/instance-kuhmgrea +[2020-11-17T11:12:58-0800] [MainThread] [I] [toil.leader] Issued job 'get_patient_fastqs' kind-get_patient_fastqs/instance-cicdq1sb with job batch system ID: 32 and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-11-17T11:12:58-0800] [MainThread] [I] [toil.leader] Issued job 'get_patient_fastqs' kind-get_patient_fastqs/instance-tyzms12q with job batch system ID: 33 and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-11-17T11:12:58-0800] [MainThread] [I] [toil.leader] Issued job 'get_patient_fastqs' kind-get_patient_fastqs/instance-3uw7w2bg with job batch system ID: 34 and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-11-17T11:12:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:12:59: Downloading Inputs for TEST +[2020-11-17T11:12:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:12:59: Obtaining file (TEST:normal_dna_fastq_2) to the file job store +[2020-11-17T11:12:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:12:59: Obtaining file (TEST:normal_dna_fastq_1) to the file job store +[2020-11-17T11:12:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:12:59: Obtaining file (TEST:tumor_dna_fastq_1) to the file job store +[2020-11-17T11:12:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:12:59: Obtaining file (TEST:tumor_rna_fastq_2) to the file job store +[2020-11-17T11:12:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:12:59: Obtaining file (TEST:tumor_rna_fastq_1) to the file job store +[2020-11-17T11:12:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:12:59: Obtaining file (TEST:tumor_dna_fastq_2) to the file job store +[2020-11-17T11:12:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpm147z6vi/worker_log.txt +[2020-11-17T11:12:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp478gp15l/worker_log.txt +[2020-11-17T11:12:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpg9kgy277/worker_log.txt +[2020-11-17T11:13:00-0800] [MainThread] [I] [toil.leader] Job ended: 'get_patient_fastqs' kind-get_patient_fastqs/instance-cicdq1sb +[2020-11-17T11:13:00-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-2k5wcr51 with job batch system ID: 35 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:13:00-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-jqmhwbe8 with job batch system ID: 36 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-17T11:13:00-0800] [MainThread] [I] [toil.leader] Job ended: 'get_patient_fastqs' kind-get_patient_fastqs/instance-3uw7w2bg +[2020-11-17T11:13:00-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-hrzipcl_ with job batch system ID: 37 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:13:00-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-eirr3b8r with job batch system ID: 38 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:13:00-0800] [MainThread] [I] [toil.leader] Job ended: 'get_patient_fastqs' kind-get_patient_fastqs/instance-tyzms12q +[2020-11-17T11:13:00-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-lp8p96ga with job batch system ID: 39 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:13:00-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-s4kmmle7 with job batch system ID: 40 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-17T11:13:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp22w31rkx/worker_log.txt +[2020-11-17T11:13:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpcg1um9gi/worker_log.txt +[2020-11-17T11:13:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3gniu3uf/worker_log.txt +[2020-11-17T11:13:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvurudee6/worker_log.txt +[2020-11-17T11:13:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp104zzl4s/worker_log.txt +[2020-11-17T11:13:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjd37jr7t/worker_log.txt +[2020-11-17T11:13:01-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-hrzipcl_ +[2020-11-17T11:13:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-etqwe7dv with job batch system ID: 41 and cores: 80, disk: 8.2 G, and memory: 2.0 G +[2020-11-17T11:13:01-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-2k5wcr51 +[2020-11-17T11:13:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-klw60gaa with job batch system ID: 42 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-17T11:13:01-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-eirr3b8r +[2020-11-17T11:13:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_cutadapt' kind-run_cutadapt/instance-cbik0xq6 with job batch system ID: 43 and cores: 1, disk: 1.8 G, and memory: 2.0 G +[2020-11-17T11:13:01-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-lp8p96ga +[2020-11-17T11:13:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-c60ojvxr with job batch system ID: 44 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-17T11:13:01-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-jqmhwbe8 +[2020-11-17T11:13:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-gyp_lxss with job batch system ID: 45 and cores: 80, disk: 13.4 G, and memory: 12.0 G +[2020-11-17T11:13:01-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-s4kmmle7 +[2020-11-17T11:13:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-qzu6cz1o with job batch system ID: 46 and cores: 80, disk: 12.8 G, and memory: 12.0 G +[2020-11-17T11:13:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp53xjgfjv/worker_log.txt +[2020-11-17T11:13:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpwxyonoux/worker_log.txt +[2020-11-17T11:13:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_cutadapt' kind-run_cutadapt/instance-cbik0xq6 +[2020-11-17T11:13:48-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-xmgyd9w3 with job batch system ID: 47 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:13:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:13:48: Ran cutadapt on TEST successfully +[2020-11-17T11:13:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8j49ywmy/worker_log.txt +[2020-11-17T11:13:57-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-etqwe7dv. +[2020-11-17T11:13:57-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-etqwe7dv +[2020-11-17T11:13:57-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-etqwe7dv +[2020-11-17T11:13:57-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-etqwe7dv follows: +=========> + [2020-11-17T11:13:01-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T11:13:01-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 59, in + nthreads=int(val) + ValueError: invalid literal for int() with base 10: '80.0' + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp53xjgfjv/ca05e503-31ea-489b-996d-90d8b8dd0e78:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_rna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '80.0']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp53xjgfjv/ca05e503-31ea-489b-996d-90d8b8dd0e78:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_rna -e /home/phlat-1.0 -o /data -p 80.0" + [2020-11-17T11:13:57-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp53xjgfjv/ca05e503-31ea-489b-996d-90d8b8dd0e78/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp53xjgfjv/ca05e503-31ea-489b-996d-90d8b8dd0e78/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp53xjgfjv/ca05e503-31ea-489b-996d-90d8b8dd0e78/phlat_index.tar.gz'} +<========= +[2020-11-17T11:13:57-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-etqwe7dv with ID kind-run_phlat/instance-etqwe7dv to 1 +[2020-11-17T11:13:57-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-etqwe7dv with job batch system ID: 48 and cores: 80, disk: 8.2 G, and memory: 2.0 G +[2020-11-17T11:13:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpcc1voyvp/worker_log.txt +[2020-11-17T11:14:48-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-c60ojvxr. +[2020-11-17T11:14:48-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-c60ojvxr +[2020-11-17T11:14:48-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-c60ojvxr +[2020-11-17T11:14:48-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-c60ojvxr follows: +=========> + [2020-11-17T11:13:49-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T11:13:49-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 59, in + nthreads=int(val) + ValueError: invalid literal for int() with base 10: '80.0' + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8j49ywmy/077a962b-4498-4272-ad50-d57b65068a85:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'normal_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '80.0']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8j49ywmy/077a962b-4498-4272-ad50-d57b65068a85:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag normal_dna -e /home/phlat-1.0 -o /data -p 80.0" + [2020-11-17T11:14:47-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8j49ywmy/077a962b-4498-4272-ad50-d57b65068a85/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8j49ywmy/077a962b-4498-4272-ad50-d57b65068a85/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8j49ywmy/077a962b-4498-4272-ad50-d57b65068a85/phlat_index.tar.gz'} +<========= +[2020-11-17T11:14:48-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-c60ojvxr with ID kind-run_phlat/instance-c60ojvxr to 1 +[2020-11-17T11:14:48-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-c60ojvxr with job batch system ID: 49 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-17T11:14:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpub9ng2zc/worker_log.txt +[2020-11-17T11:15:47-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-etqwe7dv. +[2020-11-17T11:15:47-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-etqwe7dv +[2020-11-17T11:15:47-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-etqwe7dv +[2020-11-17T11:15:47-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-etqwe7dv follows: +=========> + [2020-11-17T11:14:48-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T11:14:48-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 59, in + nthreads=int(val) + ValueError: invalid literal for int() with base 10: '80.0' + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpub9ng2zc/37416e9e-055d-4c92-8345-c2ca3d1af382:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_rna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '80.0']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpub9ng2zc/37416e9e-055d-4c92-8345-c2ca3d1af382:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_rna -e /home/phlat-1.0 -o /data -p 80.0" + [2020-11-17T11:15:46-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpub9ng2zc/37416e9e-055d-4c92-8345-c2ca3d1af382/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpub9ng2zc/37416e9e-055d-4c92-8345-c2ca3d1af382/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpub9ng2zc/37416e9e-055d-4c92-8345-c2ca3d1af382/phlat_index.tar.gz'} +<========= +[2020-11-17T11:15:47-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-etqwe7dv with ID kind-run_phlat/instance-etqwe7dv to 0 +[2020-11-17T11:15:47-0800] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-etqwe7dv with ID kind-run_phlat/instance-etqwe7dv is completely failed +[2020-11-17T11:15:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpil1gv6ny/worker_log.txt +[2020-11-17T11:16:04-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:16:04: Ran bwa on TEST:normal_dna successfully +[2020-11-17T11:16:04-0800] [MainThread] [I] [toil.leader] Job ended: 'run_bwa' kind-run_bwa/instance-qzu6cz1o +[2020-11-17T11:16:04-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-k2a__hpc with job batch system ID: 50 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:16:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp7zuz4xwv/worker_log.txt +[2020-11-17T11:16:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpu7gr9dom/worker_log.txt +[2020-11-17T11:16:05-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-xmgyd9w3 +[2020-11-17T11:16:05-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-exw4ljh3 with job batch system ID: 51 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-17T11:16:05-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-k2a__hpc +[2020-11-17T11:16:05-0800] [MainThread] [I] [toil.leader] Issued job 'bam_conversion' kind-bam_conversion/instance-8ilywiyg with job batch system ID: 52 and cores: 1, disk: 1.6 G, and memory: 12.0 G +[2020-11-17T11:16:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpes2gzpbn/worker_log.txt +[2020-11-17T11:16:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpkw61s0yx/worker_log.txt +[2020-11-17T11:16:06-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-exw4ljh3 +[2020-11-17T11:16:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_star' kind-run_star/instance-yivvztsd with job batch system ID: 53 and cores: 80, disk: 53.7 G, and memory: 48.6 G +[2020-11-17T11:16:48-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-c60ojvxr. +[2020-11-17T11:16:48-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-c60ojvxr +[2020-11-17T11:16:48-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-c60ojvxr +[2020-11-17T11:16:48-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-c60ojvxr follows: +=========> + [2020-11-17T11:15:48-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T11:15:48-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 59, in + nthreads=int(val) + ValueError: invalid literal for int() with base 10: '80.0' + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpil1gv6ny/923b58c5-a2a1-403b-8cd7-73f2ced0bf69:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'normal_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '80.0']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpil1gv6ny/923b58c5-a2a1-403b-8cd7-73f2ced0bf69:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag normal_dna -e /home/phlat-1.0 -o /data -p 80.0" + [2020-11-17T11:16:48-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpil1gv6ny/923b58c5-a2a1-403b-8cd7-73f2ced0bf69/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpil1gv6ny/923b58c5-a2a1-403b-8cd7-73f2ced0bf69/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpil1gv6ny/923b58c5-a2a1-403b-8cd7-73f2ced0bf69/phlat_index.tar.gz'} +<========= +[2020-11-17T11:16:48-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-c60ojvxr with ID kind-run_phlat/instance-c60ojvxr to 0 +[2020-11-17T11:16:48-0800] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-c60ojvxr with ID kind-run_phlat/instance-c60ojvxr is completely failed +[2020-11-17T11:16:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpytirl04g/worker_log.txt +[2020-11-17T11:17:12-0800] [MainThread] [I] [toil.leader] Job ended: 'bam_conversion' kind-bam_conversion/instance-8ilywiyg +[2020-11-17T11:17:12-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-vsa_twmp with job batch system ID: 54 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:17:12-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:17:12: Ran sam2bam on TEST:normal_dna successfully +[2020-11-17T11:17:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpwoym3fob/worker_log.txt +[2020-11-17T11:18:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:18:55: Ran bwa on TEST:tumor_dna successfully +[2020-11-17T11:18:56-0800] [MainThread] [I] [toil.leader] Job ended: 'run_bwa' kind-run_bwa/instance-gyp_lxss +[2020-11-17T11:18:56-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-4yagsm1m with job batch system ID: 55 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:18:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp32e53foh/worker_log.txt +[2020-11-17T11:18:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_1ro1iei/worker_log.txt +[2020-11-17T11:18:57-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-vsa_twmp +[2020-11-17T11:18:57-0800] [MainThread] [I] [toil.leader] Issued job 'fix_bam_header' kind-fix_bam_header/instance-ro2x1ipl with job batch system ID: 56 and cores: 1, disk: 1.6 G, and memory: 12.0 G +[2020-11-17T11:18:57-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-4yagsm1m +[2020-11-17T11:18:57-0800] [MainThread] [I] [toil.leader] Issued job 'bam_conversion' kind-bam_conversion/instance-ozphshhx with job batch system ID: 57 and cores: 1, disk: 1.7 G, and memory: 12.0 G +[2020-11-17T11:18:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpr15e5iag/worker_log.txt +[2020-11-17T11:18:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpscisbeqz/worker_log.txt +[2020-11-17T11:19:02-0800] [MainThread] [I] [toil.leader] Job ended: 'fix_bam_header' kind-fix_bam_header/instance-ro2x1ipl +[2020-11-17T11:19:02-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-h56vhhwf with job batch system ID: 58 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:19:02-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:19:02: Ran reheader on TEST:normal_dna successfully +[2020-11-17T11:19:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvc9o05w4/worker_log.txt +[2020-11-17T11:19:03-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-h56vhhwf +[2020-11-17T11:19:03-0800] [MainThread] [I] [toil.leader] Issued job 'add_readgroups' kind-add_readgroups/instance-0rk7ugd2 with job batch system ID: 59 and cores: 1, disk: 1.3 G, and memory: 12.0 G +[2020-11-17T11:19:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp2_8nv1s3/worker_log.txt +[2020-11-17T11:20:05-0800] [MainThread] [I] [toil.leader] Job ended: 'bam_conversion' kind-bam_conversion/instance-ozphshhx +[2020-11-17T11:20:05-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-4nwu1kxk with job batch system ID: 60 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:20:05-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:20:05: Ran sam2bam on TEST:tumor_dna successfully +[2020-11-17T11:20:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmph3tszgee/worker_log.txt +[2020-11-17T11:20:06-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-4nwu1kxk +[2020-11-17T11:20:06-0800] [MainThread] [I] [toil.leader] Issued job 'fix_bam_header' kind-fix_bam_header/instance-i6b4v_xd with job batch system ID: 61 and cores: 1, disk: 1.7 G, and memory: 12.0 G +[2020-11-17T11:20:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpwi1uf3bj/worker_log.txt +[2020-11-17T11:20:11-0800] [MainThread] [I] [toil.leader] Job ended: 'fix_bam_header' kind-fix_bam_header/instance-i6b4v_xd +[2020-11-17T11:20:11-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ak16v233 with job batch system ID: 62 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:20:11-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:20:11: Ran reheader on TEST:tumor_dna successfully +[2020-11-17T11:20:12-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_ub55oh9/worker_log.txt +[2020-11-17T11:20:12-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ak16v233 +[2020-11-17T11:20:12-0800] [MainThread] [I] [toil.leader] Issued job 'add_readgroups' kind-add_readgroups/instance-i2_y4xfw with job batch system ID: 63 and cores: 1, disk: 1.5 G, and memory: 12.0 G +[2020-11-17T11:20:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpd74llbli/worker_log.txt +[2020-11-17T11:20:34-0800] [MainThread] [I] [toil.leader] Job ended: 'add_readgroups' kind-add_readgroups/instance-0rk7ugd2 +[2020-11-17T11:20:34-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-mgs7llsu with job batch system ID: 64 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:20:34-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:20:34: Ran add_read_groups on TEST:normal_dna successfully +[2020-11-17T11:20:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpebw7qck0/worker_log.txt +[2020-11-17T11:20:35-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-mgs7llsu +[2020-11-17T11:20:35-0800] [MainThread] [I] [toil.leader] Issued job 'mark_duplicates' kind-mark_duplicates/instance-z4czkvcn with job batch system ID: 65 and cores: 1, disk: 1.3 G, and memory: 12.0 G +[2020-11-17T11:20:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpitng_zp_/worker_log.txt +[2020-11-17T11:21:59-0800] [MainThread] [I] [toil.leader] Job ended: 'add_readgroups' kind-add_readgroups/instance-i2_y4xfw +[2020-11-17T11:21:59-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-5ab_43re with job batch system ID: 66 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:21:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:21:59: Ran add_read_groups on TEST:tumor_dna successfully +[2020-11-17T11:21:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmprrhjzqe1/worker_log.txt +[2020-11-17T11:21:59-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-5ab_43re +[2020-11-17T11:21:59-0800] [MainThread] [I] [toil.leader] Issued job 'mark_duplicates' kind-mark_duplicates/instance-vqugr0r9 with job batch system ID: 67 and cores: 1, disk: 1.5 G, and memory: 12.0 G +[2020-11-17T11:22:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp66laoi5_/worker_log.txt +[2020-11-17T11:22:44-0800] [MainThread] [I] [toil.leader] Job ended: 'mark_duplicates' kind-mark_duplicates/instance-z4czkvcn +[2020-11-17T11:22:44-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-v3_vtmy3 with job batch system ID: 68 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:22:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:22:45: Ran mark_duplicates on TEST:normal_dna successfully +[2020-11-17T11:22:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpl5k2q1ni/worker_log.txt +[2020-11-17T11:22:45-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-v3_vtmy3 +[2020-11-17T11:22:45-0800] [MainThread] [I] [toil.leader] Issued job 'index_bamfile' kind-index_bamfile/instance-mna5sphk with job batch system ID: 69 and cores: 1, disk: 360.6 M, and memory: 12.0 G +[2020-11-17T11:22:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpyl_h3k98/worker_log.txt +[2020-11-17T11:23:09-0800] [MainThread] [I] [toil.leader] Job ended: 'index_bamfile' kind-index_bamfile/instance-mna5sphk +[2020-11-17T11:23:09-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-8o9bjndi with job batch system ID: 70 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:23:10-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:23:10: Exporting files/for-job/kind-mark_duplicates/instance-z4czkvcn/file-o9j1fbcr/normal_dna_mkdup.bam to output location +[2020-11-17T11:23:10-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:23:10: Exporting files/for-job/kind-index_bamfile/instance-mna5sphk/file-e6pgnh97/normal_dna_fix_pg_sorted.bam.bai to output location +[2020-11-17T11:23:10-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:23:10: Ran samtools-index on TEST:normal_dna successfully +[2020-11-17T11:23:10-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9t8agmh6/worker_log.txt +[2020-11-17T11:23:10-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-8o9bjndi +[2020-11-17T11:23:10-0800] [MainThread] [I] [toil.leader] Issued job 'index_bamfile' kind-Job/instance-v3_vtmy3 with job batch system ID: 71 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:23:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4jieqb5n/worker_log.txt +[2020-11-17T11:23:11-0800] [MainThread] [I] [toil.leader] Job ended: 'index_bamfile' kind-Job/instance-v3_vtmy3 +[2020-11-17T11:23:11-0800] [MainThread] [I] [toil.leader] Issued job 'mark_duplicates' kind-Job/instance-mgs7llsu with job batch system ID: 72 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:23:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpgbjiivkc/worker_log.txt +[2020-11-17T11:23:11-0800] [MainThread] [I] [toil.leader] Job ended: 'mark_duplicates' kind-Job/instance-mgs7llsu +[2020-11-17T11:23:11-0800] [MainThread] [I] [toil.leader] Issued job 'add_readgroups' kind-Job/instance-h56vhhwf with job batch system ID: 73 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:23:12-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpk5it0833/worker_log.txt +[2020-11-17T11:23:12-0800] [MainThread] [I] [toil.leader] Job ended: 'add_readgroups' kind-Job/instance-h56vhhwf +[2020-11-17T11:23:12-0800] [MainThread] [I] [toil.leader] Issued job 'fix_bam_header' kind-Job/instance-vsa_twmp with job batch system ID: 74 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:23:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpugpvha1v/worker_log.txt +[2020-11-17T11:23:13-0800] [MainThread] [I] [toil.leader] Job ended: 'fix_bam_header' kind-Job/instance-vsa_twmp +[2020-11-17T11:23:13-0800] [MainThread] [I] [toil.leader] Issued job 'bam_conversion' kind-Job/instance-k2a__hpc with job batch system ID: 75 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:23:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4mca6e18/worker_log.txt +[2020-11-17T11:23:13-0800] [MainThread] [I] [toil.leader] Job ended: 'bam_conversion' kind-Job/instance-k2a__hpc +[2020-11-17T11:23:13-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-_3vyqtvx with job batch system ID: 76 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:23:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpribgmwso/worker_log.txt +[2020-11-17T11:23:14-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-_3vyqtvx +[2020-11-17T11:24:12-0800] [MainThread] [I] [toil.leader] Job ended: 'mark_duplicates' kind-mark_duplicates/instance-vqugr0r9 +[2020-11-17T11:24:12-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-mxra1ps0 with job batch system ID: 77 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:24:12-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:24:12: Ran mark_duplicates on TEST:tumor_dna successfully +[2020-11-17T11:24:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpdjsmsw8z/worker_log.txt +[2020-11-17T11:25:17-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-klw60gaa. +[2020-11-17T11:25:17-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-klw60gaa +[2020-11-17T11:25:17-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-klw60gaa +[2020-11-17T11:25:17-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-klw60gaa follows: +=========> + [2020-11-17T11:24:13-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T11:24:13-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 59, in + nthreads=int(val) + ValueError: invalid literal for int() with base 10: '80.0' + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpdjsmsw8z/1e006918-a170-4fb9-a858-2361d5f860fb:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '80.0']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpdjsmsw8z/1e006918-a170-4fb9-a858-2361d5f860fb:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_dna -e /home/phlat-1.0 -o /data -p 80.0" + [2020-11-17T11:25:16-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpdjsmsw8z/1e006918-a170-4fb9-a858-2361d5f860fb/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpdjsmsw8z/1e006918-a170-4fb9-a858-2361d5f860fb/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpdjsmsw8z/1e006918-a170-4fb9-a858-2361d5f860fb/phlat_index.tar.gz'} +<========= +[2020-11-17T11:25:17-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-klw60gaa with ID kind-run_phlat/instance-klw60gaa to 1 +[2020-11-17T11:25:17-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-klw60gaa with job batch system ID: 78 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-17T11:25:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpthuj3zpo/worker_log.txt +[2020-11-17T11:25:18-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-mxra1ps0 +[2020-11-17T11:25:18-0800] [MainThread] [I] [toil.leader] Issued job 'index_bamfile' kind-index_bamfile/instance-v7sihqt_ with job batch system ID: 79 and cores: 1, disk: 412.9 M, and memory: 12.0 G +[2020-11-17T11:25:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvje_uqje/worker_log.txt +[2020-11-17T11:26:16-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-klw60gaa. +[2020-11-17T11:26:16-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-klw60gaa +[2020-11-17T11:26:16-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-klw60gaa +[2020-11-17T11:26:16-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-klw60gaa follows: +=========> + [2020-11-17T11:25:19-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T11:25:19-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 59, in + nthreads=int(val) + ValueError: invalid literal for int() with base 10: '80.0' + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvje_uqje/15fb74aa-d0a4-4193-8c52-42e1dc616cc8:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '80.0']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvje_uqje/15fb74aa-d0a4-4193-8c52-42e1dc616cc8:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_dna -e /home/phlat-1.0 -o /data -p 80.0" + [2020-11-17T11:26:15-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvje_uqje/15fb74aa-d0a4-4193-8c52-42e1dc616cc8/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvje_uqje/15fb74aa-d0a4-4193-8c52-42e1dc616cc8/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvje_uqje/15fb74aa-d0a4-4193-8c52-42e1dc616cc8/phlat_index.tar.gz'} +<========= +[2020-11-17T11:26:16-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-klw60gaa with ID kind-run_phlat/instance-klw60gaa to 0 +[2020-11-17T11:26:16-0800] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-klw60gaa with ID kind-run_phlat/instance-klw60gaa is completely failed +[2020-11-17T11:26:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpnlee3rgd/worker_log.txt +[2020-11-17T11:26:45-0800] [MainThread] [I] [toil.leader] Job ended: 'index_bamfile' kind-index_bamfile/instance-v7sihqt_ +[2020-11-17T11:26:45-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-igkoeo0b with job batch system ID: 80 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:26:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:26:45: Exporting files/for-job/kind-mark_duplicates/instance-vqugr0r9/file-we7wjtlu/tumor_dna_mkdup.bam to output location +[2020-11-17T11:26:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:26:45: Exporting files/for-job/kind-index_bamfile/instance-v7sihqt_/file-fvormkb0/tumor_dna_fix_pg_sorted.bam.bai to output location +[2020-11-17T11:26:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:26:45: Ran samtools-index on TEST:tumor_dna successfully +[2020-11-17T11:26:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpy4ag9vdj/worker_log.txt +[2020-11-17T11:26:46-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-igkoeo0b +[2020-11-17T11:26:46-0800] [MainThread] [I] [toil.leader] Issued job 'index_bamfile' kind-Job/instance-mxra1ps0 with job batch system ID: 81 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:26:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3uz5r7u5/worker_log.txt +[2020-11-17T11:26:47-0800] [MainThread] [I] [toil.leader] Job ended: 'index_bamfile' kind-Job/instance-mxra1ps0 +[2020-11-17T11:26:47-0800] [MainThread] [I] [toil.leader] Issued job 'mark_duplicates' kind-Job/instance-5ab_43re with job batch system ID: 82 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:26:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvrdq8esx/worker_log.txt +[2020-11-17T11:26:47-0800] [MainThread] [I] [toil.leader] Job ended: 'mark_duplicates' kind-Job/instance-5ab_43re +[2020-11-17T11:26:47-0800] [MainThread] [I] [toil.leader] Issued job 'add_readgroups' kind-Job/instance-ak16v233 with job batch system ID: 83 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:26:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3e6pk792/worker_log.txt +[2020-11-17T11:26:48-0800] [MainThread] [I] [toil.leader] Job ended: 'add_readgroups' kind-Job/instance-ak16v233 +[2020-11-17T11:26:48-0800] [MainThread] [I] [toil.leader] Issued job 'fix_bam_header' kind-Job/instance-4nwu1kxk with job batch system ID: 84 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:26:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmppa3mvyis/worker_log.txt +[2020-11-17T11:26:49-0800] [MainThread] [I] [toil.leader] Job ended: 'fix_bam_header' kind-Job/instance-4nwu1kxk +[2020-11-17T11:26:49-0800] [MainThread] [I] [toil.leader] Issued job 'bam_conversion' kind-Job/instance-4yagsm1m with job batch system ID: 85 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:26:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpdw3hcuar/worker_log.txt +[2020-11-17T11:26:49-0800] [MainThread] [I] [toil.leader] Job ended: 'bam_conversion' kind-Job/instance-4yagsm1m +[2020-11-17T11:26:49-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-j6lfkqh5 with job batch system ID: 86 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:26:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmptzj4kkn4/worker_log.txt +[2020-11-17T11:26:50-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-j6lfkqh5 +[2020-11-17T11:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-qr6ycqs7 with job batch system ID: 87 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-17T11:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-fs46wkmz with job batch system ID: 88 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-8p00djrs with job batch system ID: 89 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-y1gatlgp with job batch system ID: 90 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_indel_caller' kind-run_indel_caller/instance-zg7fs9xi with job batch system ID: 91 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp7habrhll/worker_log.txt +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8vs9lurx/worker_log.txt +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpxkp3ho8x/worker_log.txt +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3237uw_b/worker_log.txt +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpt_zmosm9/worker_log.txt +[2020-11-17T11:26:51-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:26:51: INDELs are currently unsupported.... Skipping. +[2020-11-17T11:26:51-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:26:51: Ran INDEL on TEST successfully +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Job ended: 'run_indel_caller' kind-run_indel_caller/instance-zg7fs9xi +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-8p00djrs +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-wuf__ewu with job batch system ID: 92 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-t0hrdp2k with job batch system ID: 93 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-y1gatlgp +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_strelka_full' kind-run_strelka_full/instance-wm5q0fdz with job batch system ID: 94 and cores: 25, disk: 6.0 G, and memory: 6.0 G +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-qr6ycqs7 +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-wn39lo4u with job batch system ID: 95 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-l0o4b9m5 with job batch system ID: 96 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-uyn6mtqe with job batch system ID: 97 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-zqmzqvx1 with job batch system ID: 98 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-eq9dymp6 with job batch system ID: 99 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-_mtn2smx with job batch system ID: 100 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-bcdyumhn with job batch system ID: 101 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-it77lmr1 with job batch system ID: 102 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-t3h9htf6 with job batch system ID: 103 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-35wlce3h with job batch system ID: 104 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-_rerfuh8 with job batch system ID: 105 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-r8wul16x with job batch system ID: 106 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-ve1e7qoc with job batch system ID: 107 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-qv22xcw9 with job batch system ID: 108 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-1da2pg2n with job batch system ID: 109 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-4v21o47g with job batch system ID: 110 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-2fnwk0r0 with job batch system ID: 111 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-zg9ji8gv with job batch system ID: 112 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-m450ir6j with job batch system ID: 113 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-4hetlzd_ with job batch system ID: 114 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-e4d49h11 with job batch system ID: 115 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-0x_y0kof with job batch system ID: 116 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-o04qvdp7 with job batch system ID: 117 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-9oo99q4s with job batch system ID: 118 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-5eu6tkpa with job batch system ID: 119 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-fs46wkmz +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-nog13sq5 with job batch system ID: 120 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-iqen39tg with job batch system ID: 121 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-j4l1bn40 with job batch system ID: 122 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-sq275rko with job batch system ID: 123 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-g9xa9x36 with job batch system ID: 124 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-a0cqaxkb with job batch system ID: 125 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-torev73p with job batch system ID: 126 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-m9fb_iil with job batch system ID: 127 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-e24l63l7 with job batch system ID: 128 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-h2z29s3g with job batch system ID: 129 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-no6svhib with job batch system ID: 130 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-aqzcl71n with job batch system ID: 131 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-1tmcrc4l with job batch system ID: 132 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-8empzhoa with job batch system ID: 133 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-_jo3cfrb with job batch system ID: 134 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-fgcxjw3q with job batch system ID: 135 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-tqndkqdq with job batch system ID: 136 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-wf3a831y with job batch system ID: 137 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-dg66r687 with job batch system ID: 138 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-p6bgzgg4 with job batch system ID: 139 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-9cidjtv3 with job batch system ID: 140 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-6vs9khbs with job batch system ID: 141 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-x8an7v9f with job batch system ID: 142 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-se6g0x0h with job batch system ID: 143 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:51-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-htd5l9eq with job batch system ID: 144 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpoukizq3s/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpyzc6v_m_/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_d8n2g6d/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpwqm0mhta/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-t0hrdp2k +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.leader] Issued job 'run_pileup' kind-run_pileup/instance-tnjdrxk3 with job batch system ID: 145 and cores: 1, disk: 4.8 G, and memory: 6.0 G +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfpoav1pu/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpwptnai3u/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0n3yz51t/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp1j5trnr9/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpm46357ph/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-wuf__ewu +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.leader] Issued job 'run_somaticsniper_full' kind-run_somaticsniper_full/instance-0h_b2l60 with job batch system ID: 146 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpxozt780t/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpqr2gj65v/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp80wnml66/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpt0lhf6si/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpl7aixf66/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjkc3xe23/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpzcbpbnx1/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp24m_ef7z/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpsoy2gn1x/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpnlglm49q/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp6qpnurif/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpxt1o1bw0/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpu7ixbhmt/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpcqrgx43j/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp479vdw56/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpwq1yew0g/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9epfprn3/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpkaicdvzc/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpp7cnnkzp/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpzs75s1qd/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmph58efhlo/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp2iko6jx_/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpl26vb1io/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp1tv6ztx3/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpnrgpigsp/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjxy9j_3v/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpbialeqjx/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp6gw7spvf/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjyndlu00/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpdr3zt5hd/worker_log.txt +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-_mtn2smx +[2020-11-17T11:26:53-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yb2yf65u with job batch system ID: 147 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp47h7enmb/worker_log.txt +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpi_fu4zfc/worker_log.txt +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8k29kqyw/worker_log.txt +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpakfq63et/worker_log.txt +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpkvvdju0f/worker_log.txt +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpbjnoa631/worker_log.txt +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3yn7_u6x/worker_log.txt +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp46aigwek/worker_log.txt +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpzw8mr8cp/worker_log.txt +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp7p5exxdz/worker_log.txt +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmps8d24u5e/worker_log.txt +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjkxzrj6y/worker_log.txt +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpcsfzxfpu/worker_log.txt +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp52h11a3y/worker_log.txt +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpd3pdu0oj/worker_log.txt +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpl6acx697/worker_log.txt +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpr2nfkvxw/worker_log.txt +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-t3h9htf6 +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-7_6wjfvk with job batch system ID: 148 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-l0o4b9m5 +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-aux6_f19 with job batch system ID: 149 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-uyn6mtqe +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-d9qjg3jl with job batch system ID: 150 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-eq9dymp6 +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mppqh5vo with job batch system ID: 151 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-it77lmr1 +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ge6vacd7 with job batch system ID: 152 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-bcdyumhn +[2020-11-17T11:26:54-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-q4vn5u6_ with job batch system ID: 153 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-wn39lo4u +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-uuibculn with job batch system ID: 154 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-1da2pg2n +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ewrzdiri with job batch system ID: 155 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-j4l1bn40 +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-dv63t9qv with job batch system ID: 156 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-qv22xcw9 +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9uka9hrw with job batch system ID: 157 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-zqmzqvx1 +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-eg7xdcao with job batch system ID: 158 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-0x_y0kof +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-rd57kqhd with job batch system ID: 159 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-4v21o47g +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ovr5_kcb with job batch system ID: 160 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-iqen39tg +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-_2ky5fix with job batch system ID: 161 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-2fnwk0r0 +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xbtj2suv with job batch system ID: 162 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-m450ir6j +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-v628y3w0 with job batch system ID: 163 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-_rerfuh8 +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jkvqxjgf with job batch system ID: 164 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-35wlce3h +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2u48n_5f with job batch system ID: 165 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-r8wul16x +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bv9m7hf3 with job batch system ID: 166 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-ve1e7qoc +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r1uv5wsv with job batch system ID: 167 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-zg9ji8gv +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0xdl6glf with job batch system ID: 168 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-4hetlzd_ +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-k3tx4a2p with job batch system ID: 169 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-e4d49h11 +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-06rjxk27 with job batch system ID: 170 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-o04qvdp7 +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xz7ir9sh with job batch system ID: 171 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-9oo99q4s +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r7m8gybn with job batch system ID: 172 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-5eu6tkpa +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2dtgopje with job batch system ID: 173 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-nog13sq5 +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-obc20mlb with job batch system ID: 174 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-sq275rko +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-zj_1lac9 with job batch system ID: 175 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-g9xa9x36 +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-6i952ng2 with job batch system ID: 176 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-a0cqaxkb +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-gpytfs_3 with job batch system ID: 177 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-torev73p +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-ldds2aa0 with job batch system ID: 178 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-m9fb_iil +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-xyl_w6vc with job batch system ID: 179 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-e24l63l7 +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-abto50le with job batch system ID: 180 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-h2z29s3g +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-3omj2d21 with job batch system ID: 181 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-no6svhib +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-o7el4rzn with job batch system ID: 182 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-aqzcl71n +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-7tfc1sxd with job batch system ID: 183 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-1tmcrc4l +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-clnh3tz2 with job batch system ID: 184 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-8empzhoa +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-ycwqmyaz with job batch system ID: 185 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-_jo3cfrb +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-tgl_g2py with job batch system ID: 186 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-fgcxjw3q +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-wli4u47t with job batch system ID: 187 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-tqndkqdq +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-1hxtd2ve with job batch system ID: 188 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-wf3a831y +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-z76wxbkt with job batch system ID: 189 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-dg66r687 +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-yoka6w7s with job batch system ID: 190 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-p6bgzgg4 +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-j9b505og with job batch system ID: 191 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-9cidjtv3 +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-kd9fhwhj with job batch system ID: 192 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-6vs9khbs +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-uizoz01b with job batch system ID: 193 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-x8an7v9f +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-r0hmd6ik with job batch system ID: 194 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-se6g0x0h +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-7ec5fpbf with job batch system ID: 195 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-htd5l9eq +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-diwzysne with job batch system ID: 196 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-17T11:26:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpyaa3xebo/worker_log.txt +[2020-11-17T11:26:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp373wm_lh/worker_log.txt +[2020-11-17T11:26:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp274c3vht/worker_log.txt +[2020-11-17T11:26:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpoebbz3vz/worker_log.txt +[2020-11-17T11:26:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpm26r5m7a/worker_log.txt +[2020-11-17T11:26:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmppig1hed7/worker_log.txt +[2020-11-17T11:26:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpzq9mi6ab/worker_log.txt +[2020-11-17T11:26:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphzm4yr7i/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp372uut1d/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_q89ltmb/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpy8v1h6ct/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpdd8c6091/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_3nlirfq/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpb5q5xkdk/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpyp5m8lzh/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpgdtxcrmn/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpm5_wiwwd/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpszsi60o_/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpg4rwce_h/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpb772xqwi/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmprfgbr43i/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0og4ltfc/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpc6qwo7lu/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpqpajzyqm/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_7p8avpg/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3wfz5cn_/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp275tht4_/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp36ovjz50/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpcb3oq8f8/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpr3a6h0t2/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpa58z44ke/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphxd6ga53/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpmsyi80nl/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp6z000gjf/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpw6q4c_lc/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpc330u2kc/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpyzn0a6of/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp292e3o8z/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfl5gtolb/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpmc9zckqo/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpq72c_mwx/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpwjad4xfg/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpysv4h97w/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpzvktwuqi/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpa2v7c6tp/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4kmcrtlg/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_4rv977s/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp7_n1h7xt/worker_log.txt +[2020-11-17T11:26:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp916s8f8f/worker_log.txt +[2020-11-17T11:47:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:39: Ran MuSE on TEST:chrY successfully +[2020-11-17T11:47:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:39: Ran MuSE on TEST:chrM successfully +[2020-11-17T11:47:39-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-7ec5fpbf +[2020-11-17T11:47:39-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ma5wk5l3 with job batch system ID: 197 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:40-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-diwzysne +[2020-11-17T11:47:40-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-cyhe7e92 with job batch system ID: 198 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:40: Ran MuSE on TEST:chr19 successfully +[2020-11-17T11:47:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpojmpjk1j/worker_log.txt +[2020-11-17T11:47:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:41: Ran MuSE on TEST:chr20 successfully +[2020-11-17T11:47:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-yoka6w7s +[2020-11-17T11:47:41-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-qzkvjjbs with job batch system ID: 199 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:41-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ma5wk5l3 +[2020-11-17T11:47:41-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-18llugpm with job batch system ID: 200 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpmo0g5kxj/worker_log.txt +[2020-11-17T11:47:41-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-cyhe7e92 +[2020-11-17T11:47:41-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-vhap8t3j with job batch system ID: 201 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8a2bjdup/worker_log.txt +[2020-11-17T11:47:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpcqf0lhit/worker_log.txt +[2020-11-17T11:47:42-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-j9b505og +[2020-11-17T11:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-y77qfrcp with job batch system ID: 202 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:42-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-qzkvjjbs +[2020-11-17T11:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-twclmswi with job batch system ID: 203 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpqko88old/worker_log.txt +[2020-11-17T11:47:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:42: Ran MuSE on TEST:chr22 successfully +[2020-11-17T11:47:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:42: Ran MuSE on TEST:chr21 successfully +[2020-11-17T11:47:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:42: Ran MuSE on TEST:chrX successfully +[2020-11-17T11:47:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpw50_42ok/worker_log.txt +[2020-11-17T11:47:42-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-uizoz01b +[2020-11-17T11:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-1mrbl2x9 with job batch system ID: 204 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfv9xx_lw/worker_log.txt +[2020-11-17T11:47:43-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-y77qfrcp +[2020-11-17T11:47:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-ria61kjq with job batch system ID: 205 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:43-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-kd9fhwhj +[2020-11-17T11:47:43-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-_yy10zvi with job batch system ID: 206 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:43-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-r0hmd6ik +[2020-11-17T11:47:43-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-oi5_n_zo with job batch system ID: 207 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_1fvz60y/worker_log.txt +[2020-11-17T11:47:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:43: Ran MuSE on TEST:chr15 successfully +[2020-11-17T11:47:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpnjoojri5/worker_log.txt +[2020-11-17T11:47:43-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-1mrbl2x9 +[2020-11-17T11:47:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-hwkvl88l with job batch system ID: 208 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpn4j1z0a6/worker_log.txt +[2020-11-17T11:47:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpoww50kse/worker_log.txt +[2020-11-17T11:47:44-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-_yy10zvi +[2020-11-17T11:47:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-37k0kwkx with job batch system ID: 209 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:44-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-oi5_n_zo +[2020-11-17T11:47:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-mx39byte with job batch system ID: 210 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-tgl_g2py +[2020-11-17T11:47:44-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-a_rfh0ha with job batch system ID: 211 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpb9o2708r/worker_log.txt +[2020-11-17T11:47:44-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:44: Ran MuSE on TEST:chr16 successfully +[2020-11-17T11:47:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp19otp557/worker_log.txt +[2020-11-17T11:47:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpo_enpnjd/worker_log.txt +[2020-11-17T11:47:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-wli4u47t +[2020-11-17T11:47:44-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-hdps77bm with job batch system ID: 212 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmppnrelvh4/worker_log.txt +[2020-11-17T11:47:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:45: Ran MuSE on TEST:chr18 successfully +[2020-11-17T11:47:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:45: Ran MuSE on TEST:chr17 successfully +[2020-11-17T11:47:45-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-a_rfh0ha +[2020-11-17T11:47:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-i49s7xdv with job batch system ID: 213 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjv9939o2/worker_log.txt +[2020-11-17T11:47:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-z76wxbkt +[2020-11-17T11:47:45-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-yyldhp7x with job batch system ID: 214 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:45-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-hdps77bm +[2020-11-17T11:47:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-3is6iqo7 with job batch system ID: 215 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-1hxtd2ve +[2020-11-17T11:47:45-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-1duww3up with job batch system ID: 216 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp7igqofqi/worker_log.txt +[2020-11-17T11:47:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:46: Ran MuSE on TEST:chr13 successfully +[2020-11-17T11:47:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:46: Ran MuSE on TEST:chr9 successfully +[2020-11-17T11:47:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:46: Ran MuSE on TEST:chr14 successfully +[2020-11-17T11:47:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp7i0l7awu/worker_log.txt +[2020-11-17T11:47:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp5u7rtvh5/worker_log.txt +[2020-11-17T11:47:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpx43l3kl7/worker_log.txt +[2020-11-17T11:47:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-clnh3tz2 +[2020-11-17T11:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-204p0164 with job batch system ID: 217 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:46: Ran MuSE on TEST:chr11 successfully +[2020-11-17T11:47:46-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-yyldhp7x +[2020-11-17T11:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-frwb0iel with job batch system ID: 218 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:46-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-1duww3up +[2020-11-17T11:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-b0ryazz0 with job batch system ID: 219 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-ycwqmyaz +[2020-11-17T11:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-wb1n57rk with job batch system ID: 220 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-o7el4rzn +[2020-11-17T11:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-r_bwo5aw with job batch system ID: 221 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-abto50le +[2020-11-17T11:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-7gkpox3t with job batch system ID: 222 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:47-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:47: Ran MuSE on TEST:chr10 successfully +[2020-11-17T11:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpu1fprdth/worker_log.txt +[2020-11-17T11:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpeoawmjk9/worker_log.txt +[2020-11-17T11:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpm76vrg5_/worker_log.txt +[2020-11-17T11:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmph8m15t12/worker_log.txt +[2020-11-17T11:47:47-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-204p0164 +[2020-11-17T11:47:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-8neoqggs with job batch system ID: 223 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmptmkcktiu/worker_log.txt +[2020-11-17T11:47:47-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-3omj2d21 +[2020-11-17T11:47:47-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-g3lzoxeh with job batch system ID: 224 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvxy8vh_x/worker_log.txt +[2020-11-17T11:47:47-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-wb1n57rk +[2020-11-17T11:47:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-4uibq3f4 with job batch system ID: 225 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:47-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-r_bwo5aw +[2020-11-17T11:47:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-38y7ax08 with job batch system ID: 226 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:48-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-7gkpox3t +[2020-11-17T11:47:48-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-we23ln51 with job batch system ID: 227 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:48: Ran MuSE on TEST:chr6 successfully +[2020-11-17T11:47:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:48: Ran MuSE on TEST:chr12 successfully +[2020-11-17T11:47:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:48: Ran MuSE on TEST:chr8 successfully +[2020-11-17T11:47:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0_208lia/worker_log.txt +[2020-11-17T11:47:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpmcbtzin9/worker_log.txt +[2020-11-17T11:47:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-7tfc1sxd +[2020-11-17T11:47:48-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-e4d_2mry with job batch system ID: 228 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmptqohetce/worker_log.txt +[2020-11-17T11:47:48-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-g3lzoxeh +[2020-11-17T11:47:48-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-nxq0uvyv with job batch system ID: 229 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-gpytfs_3 +[2020-11-17T11:47:48-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-rjbvx3cb with job batch system ID: 230 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-xyl_w6vc +[2020-11-17T11:47:48-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-rufpw59p with job batch system ID: 231 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpiv0t3p5n/worker_log.txt +[2020-11-17T11:47:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpb1hux_wy/worker_log.txt +[2020-11-17T11:47:49-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:49: Ran MuSE on TEST:chr5 successfully +[2020-11-17T11:47:49-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:49: Ran MuSE on TEST:chr3 successfully +[2020-11-17T11:47:49-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:49: Ran MuSE on TEST:chr7 successfully +[2020-11-17T11:47:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp7qzyfo_x/worker_log.txt +[2020-11-17T11:47:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmptgrnl5se/worker_log.txt +[2020-11-17T11:47:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvuwzcyck/worker_log.txt +[2020-11-17T11:47:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpgbqwnqa7/worker_log.txt +[2020-11-17T11:47:49-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-dv63t9qv +[2020-11-17T11:47:49-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-bt0w5mme with job batch system ID: 232 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:49-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-6i952ng2 +[2020-11-17T11:47:49-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-kuw1k779 with job batch system ID: 233 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:49-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-ldds2aa0 +[2020-11-17T11:47:49-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-qcpencxv with job batch system ID: 234 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:49-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:49: Exporting files/for-job/kind-run_star/instance-yivvztsd/file-uf_aq3um/rnaAligned.toTranscriptome.out.bam to output location +[2020-11-17T11:47:49-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:49: Exporting files/for-job/kind-run_star/instance-yivvztsd/file-s1ylhwzy/rnaChimeric.out.junction to output location +[2020-11-17T11:47:49-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:49: Ran STAR on TEST successfully +[2020-11-17T11:47:49-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:49: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-exw4ljh3/cleanup/file-z1fjynkz/stream used 112.45% (60.3 GB [64780677120B] used, 53.7 GB [57610702127B] requested) at the end of its run. +[2020-11-17T11:47:49-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:49: Ran MuSE on TEST:chr4 successfully +[2020-11-17T11:47:49-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-rjbvx3cb +[2020-11-17T11:47:49-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-8ia_0vsw with job batch system ID: 235 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:49-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-e4d_2mry +[2020-11-17T11:47:49-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-l8k5vyv8 with job batch system ID: 236 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:49-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-rufpw59p +[2020-11-17T11:47:49-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-h7a36bf2 with job batch system ID: 237 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:50-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-zj_1lac9 +[2020-11-17T11:47:50-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-yrlo2ieu with job batch system ID: 238 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphj0zlfnv/worker_log.txt +[2020-11-17T11:47:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpwwxgxvbq/worker_log.txt +[2020-11-17T11:47:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpz0toblwp/worker_log.txt +[2020-11-17T11:47:50-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-bt0w5mme +[2020-11-17T11:47:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-gp6jkzck with job batch system ID: 239 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpl6rn3y7v/worker_log.txt +[2020-11-17T11:47:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpx985pjub/worker_log.txt +[2020-11-17T11:47:50-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-qcpencxv +[2020-11-17T11:47:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-e73nfut4 with job batch system ID: 240 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:50-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-kuw1k779 +[2020-11-17T11:47:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-iud4w5pc with job batch system ID: 241 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphivxa4aj/worker_log.txt +[2020-11-17T11:47:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:50: Ran MuSE on TEST:chr2 successfully +[2020-11-17T11:47:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpcyodzmc_/worker_log.txt +[2020-11-17T11:47:50-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-yrlo2ieu +[2020-11-17T11:47:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-_5jv2w3r with job batch system ID: 242 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp96_eymvk/worker_log.txt +[2020-11-17T11:47:51-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-_2ky5fix +[2020-11-17T11:47:51-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-6njncht8 with job batch system ID: 243 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpca9lg_ba/worker_log.txt +[2020-11-17T11:47:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp1od9qfh_/worker_log.txt +[2020-11-17T11:47:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpyz_estcq/worker_log.txt +[2020-11-17T11:47:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfi89ops7/worker_log.txt +[2020-11-17T11:47:51-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-6njncht8 +[2020-11-17T11:47:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-nj7m1ef6 with job batch system ID: 244 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:52-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpcc9oqmn7/worker_log.txt +[2020-11-17T11:47:56-0800] [MainThread] [I] [toil.leader] Job ended: 'run_star' kind-run_star/instance-yivvztsd +[2020-11-17T11:47:56-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-7b4t104b with job batch system ID: 245 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:56: Ran MuSE on TEST:chr1 successfully +[2020-11-17T11:47:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpq7nou6k2/worker_log.txt +[2020-11-17T11:47:57-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-7b4t104b +[2020-11-17T11:47:57-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-4e3t4erp with job batch system ID: 246 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:47:57-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-obc20mlb +[2020-11-17T11:47:57-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-img3uaw0 with job batch system ID: 247 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfs1g7prp/worker_log.txt +[2020-11-17T11:47:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmptbi1xtmj/worker_log.txt +[2020-11-17T11:47:58-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-img3uaw0 +[2020-11-17T11:47:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-ac_xcshl with job batch system ID: 248 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-17T11:47:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmppn3_v6p9/worker_log.txt +[2020-11-17T11:47:58-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-vhap8t3j +[2020-11-17T11:47:58-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-byv_0kbl with job batch system ID: 249 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:59: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-vhap8t3j/file-1dsa2do4/chrM.vcf to output location +[2020-11-17T11:47:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:59: Ran MuSE sump on TEST:chrM successfully +[2020-11-17T11:47:59-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-18llugpm +[2020-11-17T11:47:59-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-1_funw33 with job batch system ID: 250 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmplqlgae8w/worker_log.txt +[2020-11-17T11:47:59-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-byv_0kbl +[2020-11-17T11:47:59-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-cyhe7e92 with job batch system ID: 251 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:59-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-ria61kjq +[2020-11-17T11:47:59-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-kvuo84l9 with job batch system ID: 252 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:59: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-twclmswi/file-1hssr9_p/chr19.vcf to output location +[2020-11-17T11:47:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:59: Ran MuSE sump on TEST:chr19 successfully +[2020-11-17T11:47:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:59: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-37k0kwkx/file-hz1xi2ze/chr21.vcf to output location +[2020-11-17T11:47:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:59: Ran MuSE sump on TEST:chr21 successfully +[2020-11-17T11:47:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:59: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-18llugpm/file-tfviin8e/chrY.vcf to output location +[2020-11-17T11:47:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:59: Ran MuSE sump on TEST:chrY successfully +[2020-11-17T11:47:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:59: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-hwkvl88l/file-e2osztz4/chr22.vcf to output location +[2020-11-17T11:47:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:59: Ran MuSE sump on TEST:chr22 successfully +[2020-11-17T11:47:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:59: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-ria61kjq/file-91qvxvaf/chr20.vcf to output location +[2020-11-17T11:47:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:47:59: Ran MuSE sump on TEST:chr20 successfully +[2020-11-17T11:47:59-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-37k0kwkx +[2020-11-17T11:47:59-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-sl4w6id8 with job batch system ID: 253 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:59-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-hwkvl88l +[2020-11-17T11:47:59-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-r2jq9gk8 with job batch system ID: 254 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:59-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-twclmswi +[2020-11-17T11:47:59-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-xq9xc_ka with job batch system ID: 255 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:59-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-mx39byte +[2020-11-17T11:47:59-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-wg1czvht with job batch system ID: 256 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:59-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-i49s7xdv +[2020-11-17T11:47:59-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-kigea3dx with job batch system ID: 257 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:47:59-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-8neoqggs +[2020-11-17T11:47:59-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-0pntzcj6 with job batch system ID: 258 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp61sahlbx/worker_log.txt +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-3is6iqo7 +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-hfvvzojv with job batch system ID: 259 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-b0ryazz0 +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-1cqdocru with job batch system ID: 260 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-1_funw33 +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-ma5wk5l3 with job batch system ID: 261 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-8ia_0vsw +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ofi9oi92 with job batch system ID: 262 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-we23ln51 +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-fkrl12fn with job batch system ID: 263 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-l8k5vyv8 +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-6bntup3o with job batch system ID: 264 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-l8k5vyv8/file-lditsv1s/chr12.vcf to output location +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Ran MuSE sump on TEST:chr12 successfully +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-mx39byte/file-ziv_7jm8/chrX.vcf to output location +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Ran MuSE sump on TEST:chrX successfully +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-i49s7xdv/file-9lbr6cvc/chr15.vcf to output location +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Ran MuSE sump on TEST:chr15 successfully +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-8neoqggs/file-kbk__6k1/chr13.vcf to output location +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Ran MuSE sump on TEST:chr13 successfully +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-3is6iqo7/file-x475skft/chr16.vcf to output location +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Ran MuSE sump on TEST:chr16 successfully +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-b0ryazz0/file-jy00e2ia/chr17.vcf to output location +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Ran MuSE sump on TEST:chr17 successfully +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-8ia_0vsw/file-57wvzvzw/chr6.vcf to output location +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Ran MuSE sump on TEST:chr6 successfully +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-we23ln51/file-pu13n2d4/chr9.vcf to output location +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Ran MuSE sump on TEST:chr9 successfully +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-4uibq3f4 +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-squigke8 with job batch system ID: 265 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-nxq0uvyv +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-st_eyyj3 with job batch system ID: 266 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-frwb0iel +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-t9gtgyn7 with job batch system ID: 267 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpoucf3u8z/worker_log.txt +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3ke_kx2d/worker_log.txt +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-gp6jkzck +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ddxihwi9 with job batch system ID: 268 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpuk9kvu4p/worker_log.txt +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-38y7ax08 +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-fg_52mis with job batch system ID: 269 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-cyhe7e92 +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-htd5l9eq with job batch system ID: 270 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-h7a36bf2 +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-0nul9chf with job batch system ID: 271 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-kvuo84l9 +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-y77qfrcp with job batch system ID: 272 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-sl4w6id8 +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-_yy10zvi with job batch system ID: 273 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjxk3r6jl/worker_log.txt +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpcldz3fmj/worker_log.txt +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-e73nfut4 +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-7btb3bjv with job batch system ID: 274 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpzasatf7j/worker_log.txt +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpt_z2jk8f/worker_log.txt +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-_5jv2w3r +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-hqg4bl1o with job batch system ID: 275 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-iud4w5pc +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-v7l86e6q with job batch system ID: 276 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-e73nfut4/file-o6havvsx/chr7.vcf to output location +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Ran MuSE sump on TEST:chr7 successfully +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-frwb0iel/file-8hxp62nn/chr18.vcf to output location +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Ran MuSE sump on TEST:chr18 successfully +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-nj7m1ef6/file-7omnpd1m/chr2.vcf to output location +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Ran MuSE sump on TEST:chr2 successfully +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-nxq0uvyv/file-b8mpdkhi/chr10.vcf to output location +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Ran MuSE sump on TEST:chr10 successfully +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-gp6jkzck/file-ofrpucdr/chr3.vcf to output location +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Ran MuSE sump on TEST:chr3 successfully +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-38y7ax08/file-zuaymx63/chr11.vcf to output location +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Ran MuSE sump on TEST:chr11 successfully +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-h7a36bf2/file-yjgd9lqd/chr8.vcf to output location +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Ran MuSE sump on TEST:chr8 successfully +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-_5jv2w3r/file-jnc0itx7/chr4.vcf to output location +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Ran MuSE sump on TEST:chr4 successfully +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-4uibq3f4/file-eyd_jted/chr14.vcf to output location +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Ran MuSE sump on TEST:chr14 successfully +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-iud4w5pc/file-esad182o/chr5.vcf to output location +[2020-11-17T11:48:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:00: Ran MuSE sump on TEST:chr5 successfully +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-nj7m1ef6 +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-odk4ztyw with job batch system ID: 277 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp1qc1vy8t/worker_log.txt +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-r2jq9gk8 +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-1mrbl2x9 with job batch system ID: 278 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-xq9xc_ka +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-qzkvjjbs with job batch system ID: 279 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp6_wxs1fv/worker_log.txt +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-kigea3dx +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-a_rfh0ha with job batch system ID: 280 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-wg1czvht +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-oi5_n_zo with job batch system ID: 281 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-0pntzcj6 +[2020-11-17T11:48:00-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-204p0164 with job batch system ID: 282 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpz1k0w4ok/worker_log.txt +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3do1vu6g/worker_log.txt +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-hfvvzojv +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-hdps77bm with job batch system ID: 283 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpah9i5vg4/worker_log.txt +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpf4fo_27q/worker_log.txt +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-ma5wk5l3 +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-se6g0x0h with job batch system ID: 284 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpoxp9owjx/worker_log.txt +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ofi9oi92 +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-rjbvx3cb with job batch system ID: 285 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-fkrl12fn +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-7gkpox3t with job batch system ID: 286 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-1cqdocru +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-1duww3up with job batch system ID: 287 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-6bntup3o +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-e4d_2mry with job batch system ID: 288 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8uqdpjxp/worker_log.txt +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0uby1pfw/worker_log.txt +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpr1_ivkeh/worker_log.txt +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-squigke8 +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-wb1n57rk with job batch system ID: 289 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpx4peild6/worker_log.txt +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ddxihwi9 +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-bt0w5mme with job batch system ID: 290 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpbhvq66mc/worker_log.txt +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-t9gtgyn7 +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpwne5ksgz/worker_log.txt +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-yyldhp7x with job batch system ID: 291 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-st_eyyj3 +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-g3lzoxeh with job batch system ID: 292 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpy6ddj0rv/worker_log.txt +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpt906i8n3/worker_log.txt +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpbx2q1dey/worker_log.txt +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-_yy10zvi +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-9cidjtv3 with job batch system ID: 293 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpelxfrwr2/worker_log.txt +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-htd5l9eq +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-0nul9chf +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-rufpw59p with job batch system ID: 294 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4dxwmfwa/worker_log.txt +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpu359n49p/worker_log.txt +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp6bcdyexf/worker_log.txt +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-fg_52mis +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-r_bwo5aw with job batch system ID: 295 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-y77qfrcp +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-p6bgzgg4 with job batch system ID: 296 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpo166yd0w/worker_log.txt +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-hqg4bl1o +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-yrlo2ieu with job batch system ID: 297 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpn4w_4sr6/worker_log.txt +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmph5wb7s3x/worker_log.txt +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-qzkvjjbs +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-dg66r687 with job batch system ID: 298 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-7btb3bjv +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-qcpencxv with job batch system ID: 299 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpit75vtzs/worker_log.txt +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-1mrbl2x9 +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-6vs9khbs with job batch system ID: 300 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-v7l86e6q +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-kuw1k779 with job batch system ID: 301 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpe5d1aex4/worker_log.txt +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-odk4ztyw +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-6njncht8 with job batch system ID: 302 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-a_rfh0ha +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-_jo3cfrb with job batch system ID: 303 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpmn6v3623/worker_log.txt +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-204p0164 +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-1tmcrc4l with job batch system ID: 304 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpp403t6v5/worker_log.txt +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-oi5_n_zo +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-x8an7v9f with job batch system ID: 305 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpcux4_30p/worker_log.txt +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpdo4wwqy0/worker_log.txt +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-se6g0x0h +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpnjtufbr5/worker_log.txt +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-hdps77bm +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-fgcxjw3q with job batch system ID: 306 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-rjbvx3cb +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-a0cqaxkb with job batch system ID: 307 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpmr0wh6iv/worker_log.txt +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-7gkpox3t +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-e24l63l7 with job batch system ID: 308 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-e4d_2mry +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-aqzcl71n with job batch system ID: 309 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-1duww3up +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-tqndkqdq with job batch system ID: 310 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphd6p9f_4/worker_log.txt +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4ea40gpp/worker_log.txt +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-wb1n57rk +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-8empzhoa with job batch system ID: 311 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-bt0w5mme +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-j4l1bn40 with job batch system ID: 312 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpzb4glbl9/worker_log.txt +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpao7t7wxi/worker_log.txt +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-yyldhp7x +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-wf3a831y with job batch system ID: 313 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-g3lzoxeh +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-h2z29s3g with job batch system ID: 314 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp2jkrvyo6/worker_log.txt +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpu_uei8eq/worker_log.txt +[2020-11-17T11:48:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpnb5v4pyo/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-ac_xcshl +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-usat2xyy with job batch system ID: 315 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-rufpw59p +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-m9fb_iil with job batch system ID: 316 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9fc1dh42/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-9cidjtv3 +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-r_bwo5aw +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-no6svhib with job batch system ID: 317 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpemf5psdx/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpkur1v90q/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp6767zh01/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpkp_14y55/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-yrlo2ieu +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-sq275rko with job batch system ID: 318 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpq1ecsmga/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-p6bgzgg4 +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-qcpencxv +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-torev73p with job batch system ID: 319 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpdk1x8atm/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-6vs9khbs +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-kuw1k779 +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-g9xa9x36 with job batch system ID: 320 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmppneltlyh/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp7ce7ijtm/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-6njncht8 +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-iqen39tg with job batch system ID: 321 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-1tmcrc4l +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpha0jbyr1/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp__77vqs3/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-_jo3cfrb +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-dg66r687 +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp1snu24xe/worker_log.txt +[2020-11-17T11:48:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:03: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-ac_xcshl/file-j7v8w8m9/chr1.vcf to output location +[2020-11-17T11:48:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:03: Ran MuSE sump on TEST:chr1 successfully +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-a0cqaxkb +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpbv2xr6qw/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-x8an7v9f +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpr6fxzp67/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-fgcxjw3q +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-aqzcl71n +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpglihv3rq/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-e24l63l7 +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmplwlawl0o/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-tqndkqdq +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpz5jxb5bf/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-j4l1bn40 +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-8empzhoa +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmplf52i6fq/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpv0ycpz1l/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpbfy64hkc/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-m9fb_iil +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp75obya1g/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpd1h4yib1/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-wf3a831y +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-h2z29s3g +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpk2aiauaa/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-no6svhib +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-usat2xyy +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-img3uaw0 with job batch system ID: 322 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpq3ij32k0/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp679tv4b9/worker_log.txt +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-sq275rko +[2020-11-17T11:48:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3vaqrqgm/worker_log.txt +[2020-11-17T11:48:04-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-torev73p +[2020-11-17T11:48:04-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-g9xa9x36 +[2020-11-17T11:48:04-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-iqen39tg +[2020-11-17T11:48:04-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjdwwbufh/worker_log.txt +[2020-11-17T11:48:04-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-img3uaw0 +[2020-11-17T11:48:04-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-nog13sq5 with job batch system ID: 323 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpaopntgw_/worker_log.txt +[2020-11-17T11:48:05-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-nog13sq5 +[2020-11-17T11:48:05-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-xjh1jzz7 with job batch system ID: 324 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp5fag6f7n/worker_log.txt +[2020-11-17T11:48:06-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-xjh1jzz7 +[2020-11-17T11:48:31-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-4e3t4erp +[2020-11-17T11:48:31-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-7b4t104b with job batch system ID: 325 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:31: Ran samtools-sort on TEST:rna successfully +[2020-11-17T11:48:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:31: Exporting files/for-job/kind-EncapsulatedJob/instance-4e3t4erp/file-3rxu298t/rna_sorted.bam to output location +[2020-11-17T11:48:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:31: Exporting files/for-job/kind-EncapsulatedJob/instance-4e3t4erp/file-wdamje6x/rna_genome_sorted.bam.bai to output location +[2020-11-17T11:48:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:31: Ran samtools-index on TEST:rna successfully +[2020-11-17T11:48:31-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmplunl1kl8/worker_log.txt +[2020-11-17T11:48:31-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-7b4t104b +[2020-11-17T11:48:31-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-6biujoqf with job batch system ID: 326 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp97vtk5k3/worker_log.txt +[2020-11-17T11:48:32-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-6biujoqf +[2020-11-17T11:48:32-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-v67rsjos with job batch system ID: 327 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-17T11:48:32-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-3nats6mz with job batch system ID: 328 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-17T11:48:32-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-dbtlmgmn with job batch system ID: 329 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-17T11:48:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpb6znsipe/worker_log.txt +[2020-11-17T11:48:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpraqnmhs_/worker_log.txt +[2020-11-17T11:48:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjx6xhcyf/worker_log.txt +[2020-11-17T11:48:33-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-3nats6mz +[2020-11-17T11:48:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with job batch system ID: 330 and cores: 80, disk: 5.0 G, and memory: 2.0 G +[2020-11-17T11:48:33-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-v67rsjos +[2020-11-17T11:48:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_fusion' kind-run_fusion/instance-g9_082sr with job batch system ID: 331 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-17T11:48:33-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-dbtlmgmn +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-y0a38ryy with job batch system ID: 332 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-9e1gplto with job batch system ID: 333 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-3u67g6qc with job batch system ID: 334 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-kl1aviz3 with job batch system ID: 335 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-vajvu1ke with job batch system ID: 336 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-hyoqamqz with job batch system ID: 337 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-rut4j81p with job batch system ID: 338 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-n107zzip with job batch system ID: 339 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-pzkcodc4 with job batch system ID: 340 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-ncemqmds with job batch system ID: 341 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-heu34ntv with job batch system ID: 342 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-2gnawt2n with job batch system ID: 343 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-pmk6jqh0 with job batch system ID: 344 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-rasdqp_4 with job batch system ID: 345 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-x5vr8lls with job batch system ID: 346 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-9v6yyeiq with job batch system ID: 347 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-71zlpdlm with job batch system ID: 348 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-6ja23ir9 with job batch system ID: 349 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-x7cm8te9 with job batch system ID: 350 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-urgdyniq with job batch system ID: 351 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-4q5d86eb with job batch system ID: 352 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-o08r_0ko with job batch system ID: 353 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-3npacuv2 with job batch system ID: 354 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-l8a5rrhr with job batch system ID: 355 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-8af2a19g with job batch system ID: 356 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:48:34-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:34: Ran spawn_radia on TEST successfully +[2020-11-17T11:48:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpqb2aj7b1/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphpmeehvz/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpscpjiwc0/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpl7g4k3h2/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp2tyj_5gf/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmplhajhe8w/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpcqngpbx5/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpwjvn3wcj/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpw7jd2qhx/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfvmygptv/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmptyc1y_k5/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9mb2h3e3/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpl1qnv5xv/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpheiw7p66/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp82pyho_k/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp62eajqkm/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp26i2dfk4/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp6xpd9bx3/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmputlml4ux/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfc1x83yb/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjh8_wvhl/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpb5f4yktx/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvm7b3jgj/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9vq7nm_o/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjbi2q8kj/worker_log.txt +[2020-11-17T11:48:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpqo7yt_32/worker_log.txt +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-3u67g6qc +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-31_tfqh9 with job batch system ID: 357 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-hyoqamqz +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-sk1hffpu with job batch system ID: 358 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-rasdqp_4 +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-_7k4bhmw with job batch system ID: 359 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-y0a38ryy +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-1ixkm7sa with job batch system ID: 360 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-kl1aviz3 +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-y3155cnv with job batch system ID: 361 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-n107zzip +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-tylx98yd with job batch system ID: 362 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-71zlpdlm +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-l9gd4jyv with job batch system ID: 363 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-vajvu1ke +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-fexr4ne1 with job batch system ID: 364 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-rut4j81p +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-qomyhl4z with job batch system ID: 365 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-heu34ntv +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-uuz6egk3 with job batch system ID: 366 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-2gnawt2n +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-ih1hr4ee with job batch system ID: 367 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-pzkcodc4 +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-t4meaiwm with job batch system ID: 368 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-x5vr8lls +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-bh6j1ee8 with job batch system ID: 369 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-9e1gplto +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-c6mc4yyt with job batch system ID: 370 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-pmk6jqh0 +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-l0fvnl_y with job batch system ID: 371 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-ncemqmds +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-ozzpit9s with job batch system ID: 372 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-9v6yyeiq +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-8df115wr with job batch system ID: 373 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-6ja23ir9 +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-yk7gttg2 with job batch system ID: 374 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-x7cm8te9 +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-s9p3b8oc with job batch system ID: 375 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-urgdyniq +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-aoi37xzh with job batch system ID: 376 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-4q5d86eb +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-ltaggypj with job batch system ID: 377 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-o08r_0ko +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-8y_bwf3d with job batch system ID: 378 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-3npacuv2 +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-le1j906_ with job batch system ID: 379 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-l8a5rrhr +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-9_85_oup with job batch system ID: 380 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-8af2a19g +[2020-11-17T11:48:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-l2grlbre with job batch system ID: 381 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpb1qh0w81/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpr33n26op/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpekxmhzs1/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp38skaqyj/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpg5yslss_/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmph97zsevz/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpu8tuqr16/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpxenvm4wo/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpd0uasj0j/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpyfbf9n9s/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp7lj0qvc5/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpujb_cqss/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpbt982edl/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpqaq1ukrr/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8o_quy0t/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpw6h62ctf/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpxynvpcj3/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3voptpsx/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8f0yxylh/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpi9igopvt/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9z0q62l3/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpe5vfe44c/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpa6y9_kh4/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpya5mvexc/worker_log.txt +[2020-11-17T11:48:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpe_b4q6pt/worker_log.txt +[2020-11-17T11:48:53-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:48:53: Ran samtools pileup on TEST successfully +[2020-11-17T11:48:54-0800] [MainThread] [I] [toil.leader] Job ended: 'run_pileup' kind-run_pileup/instance-tnjdrxk3 +[2020-11-17T11:48:54-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-r9pjl3mp with job batch system ID: 382 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:48:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8hwnp957/worker_log.txt +[2020-11-17T11:48:55-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-r9pjl3mp +[2020-11-17T11:51:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:51:55: Ran SomaticSniper on TEST successfully +[2020-11-17T11:51:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:51:55: Ran strelka on TEST successfully +[2020-11-17T11:51:56-0800] [MainThread] [I] [toil.leader] Job ended: 'run_somaticsniper_full' kind-run_somaticsniper_full/instance-0h_b2l60 +[2020-11-17T11:51:56-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-nxi6lpyj with job batch system ID: 383 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:51:56-0800] [MainThread] [I] [toil.leader] Job ended: 'run_strelka_full' kind-run_strelka_full/instance-wm5q0fdz +[2020-11-17T11:51:56-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ca8sfwbb with job batch system ID: 384 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:51:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp13j8yr8c/worker_log.txt +[2020-11-17T11:51:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpzyiq4e9h/worker_log.txt +[2020-11-17T11:51:57-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ca8sfwbb +[2020-11-17T11:51:57-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-833kw86t with job batch system ID: 385 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:51:57-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-nxi6lpyj +[2020-11-17T11:51:57-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-to8zefli with job batch system ID: 386 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:51:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3okv0w4g/worker_log.txt +[2020-11-17T11:51:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpz9icrd9y/worker_log.txt +[2020-11-17T11:51:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-833kw86t +[2020-11-17T11:51:58-0800] [MainThread] [I] [toil.leader] Issued job 'unmerge' kind-unmerge/instance-pcldxh6h with job batch system ID: 387 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:51:58-0800] [MainThread] [I] [toil.leader] Issued job 'unmerge' kind-unmerge/instance-nww2_och with job batch system ID: 388 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:51:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-to8zefli +[2020-11-17T11:51:58-0800] [MainThread] [I] [toil.leader] Issued job 'filter_somaticsniper' kind-filter_somaticsniper/instance-27dvr2ro with job batch system ID: 389 and cores: 1, disk: 4.8 G, and memory: 6.0 G +[2020-11-17T11:51:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvvjxxqpg/worker_log.txt +[2020-11-17T11:51:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpqzawzh1e/worker_log.txt +[2020-11-17T11:51:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpe4b9t6jo/worker_log.txt +[2020-11-17T11:52:01-0800] [MainThread] [I] [toil.leader] Job ended: 'unmerge' kind-unmerge/instance-pcldxh6h +[2020-11-17T11:52:01-0800] [MainThread] [I] [toil.leader] Job ended: 'unmerge' kind-unmerge/instance-nww2_och +[2020-11-17T11:52:01-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-x0p554j1 with job batch system ID: 390 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-h7kk7u9a/chr3.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-vif44s01/chrM.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-rf3mgevc/chr8.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-_mly3e4w/chr10.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-45onetq8/chr18.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-3bdnum5x/chr6.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-nuxbc5fe/chr15.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-wcxqcee_/chr16.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-yzmu3v1f/chr4.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-8gth4khw/chr11.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-beflxop8/chr19.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-q923c9jo/chr17.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-vj9son3c/chr14.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-ibmfoyvb/chr13.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-jvzdjgnx/chr7.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-1kahrakp/chr5.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-c61kgbtb/chrY.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-2fl8x0jt/chr1.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-4025x5ty/chr9.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-o88ysl2l/chr22.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-ebzm8e47/chr12.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-23ffj8hx/chr20.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-86cywx2w/chr2.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-ft7efi9_/chrX.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-pcldxh6h/file-96h5o8cc/chr21.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-ccigv1n1/chr22.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-pp2mcy05/chr21.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-b4nl3wos/chr1.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-0ktyep4k/chr19.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-c49o962t/chr6.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-07vr8x6t/chr20.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-w76qmxqz/chr14.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-x8of8cyb/chrX.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-g_yvosmj/chr8.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-o4yxeh4d/chr12.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-y_d2csd1/chrY.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-pow0prkt/chr7.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-516_j1xs/chr17.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-zo0rsd4y/chr13.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-zhtbxgpm/chr9.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-9x_vy9o9/chr4.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-o9q5i5nd/chr11.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-qdee8_we/chr2.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-no_p9ygp/chr15.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-2y3mc7wq/chr3.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-f3p8563b/chr16.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-r2dv52vc/chrM.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-m6823vfx/chr10.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-sjblo7m1/chr5.vcf to output location +[2020-11-17T11:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:52:01: Exporting files/for-job/kind-unmerge/instance-nww2_och/file-st9c2aib/chr18.vcf to output location +[2020-11-17T11:52:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9trujyse/worker_log.txt +[2020-11-17T11:52:02-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-x0p554j1 +[2020-11-17T11:52:02-0800] [MainThread] [I] [toil.leader] Issued job 'wrap_unmerge' kind-EncapsulatedJob/instance-833kw86t with job batch system ID: 391 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:52:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpukqeo_mt/worker_log.txt +[2020-11-17T11:52:03-0800] [MainThread] [I] [toil.leader] Job ended: 'wrap_unmerge' kind-EncapsulatedJob/instance-833kw86t +[2020-11-17T11:52:03-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ca8sfwbb with job batch system ID: 392 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:52:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpawani_xk/worker_log.txt +[2020-11-17T11:52:03-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ca8sfwbb +[2020-11-17T11:52:03-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ll4bb717 with job batch system ID: 393 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:52:04-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpkr0tc7v9/worker_log.txt +[2020-11-17T11:52:04-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ll4bb717 +[2020-11-17T11:52:05-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_rsem file:/scratch/drkthomp/jobStore kind-run_rsem/instance-mzzok2e3. +[2020-11-17T11:52:05-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-17T11:52:05-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-17T11:52:05-0800] [MainThread] [W] [toil.leader] Log from job kind-run_rsem/instance-mzzok2e3 follows: +=========> + [2020-11-17T11:48:34-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T11:48:34-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + 1.2.20: Pulling from aarjunrao/rsem + 943c334059c7: Already exists + a1acf99303d2: Already exists + 27616aacb7b3: Already exists + 35d12cd1c9fc: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + a27d4604b3b3: Already exists + b23184322fd8: Already exists + 45490e2c3ebf: Already exists + a3ed95caeb02: Already exists + 07bc45a5d3f9: Already exists + a3ed95caeb02: Already exists + 4e8282f9b0b2: Already exists + d8467ea2e124: Already exists + b53fc7d0498f: Already exists + dddf0f2e7c16: Pulling fs layer + a879e59ad650: Pulling fs layer + 67e4b34365f4: Pulling fs layer + 5f5aaf55fb95: Pulling fs layer + a3ed95caeb02: Pulling fs layer + 5f5aaf55fb95: Waiting + a3ed95caeb02: Waiting + 67e4b34365f4: Verifying Checksum + 67e4b34365f4: Download complete + 5f5aaf55fb95: Verifying Checksum + 5f5aaf55fb95: Download complete + a3ed95caeb02: Download complete + a879e59ad650: Verifying Checksum + a879e59ad650: Download complete + dddf0f2e7c16: Verifying Checksum + dddf0f2e7c16: Download complete + dddf0f2e7c16: Pull complete + a879e59ad650: Pull complete + 67e4b34365f4: Pull complete + 5f5aaf55fb95: Pull complete + a3ed95caeb02: Pull complete + Digest: sha256:cd6187e8ff78cdc1d513997680d30c473231020afd4834e55da6e2c64e825016 + Status: Downloaded newer image for aarjunrao/rsem:1.2.20 + Value "80.0" invalid for option p (number expected) + NAME + rsem-calculate-expression + + SYNOPSIS + rsem-calculate-expression [options] upstream_read_file(s) reference_name sample_name + rsem-calculate-expression [options] --paired-end upstream_read_file(s) downstream_read_file(s) reference_name sample_name + rsem-calculate-expression [options] --sam/--bam [--paired-end] input reference_name sample_name + + ARGUMENTS + upstream_read_files(s) + Comma-separated list of files containing single-end reads or + upstream reads for paired-end data. By default, these files are + assumed to be in FASTQ format. If the --no-qualities option is + specified, then FASTA format is expected. + + downstream_read_file(s) + Comma-separated list of files containing downstream reads which are + paired with the upstream reads. By default, these files are assumed + to be in FASTQ format. If the --no-qualities option is specified, + then FASTA format is expected. + + input + SAM/BAM formatted input file. If "-" is specified for the filename, + SAM/BAM input is instead assumed to come from standard input. RSEM + requires all alignments of the same read group together. For + paired-end reads, RSEM also requires the two mates of any alignment + be adjacent. See Description section for how to make input file obey + RSEM's requirements. + + reference_name + The name of the reference used. The user must have run + 'rsem-prepare-reference' with this reference_name before running + this program. + + sample_name + The name of the sample analyzed. All output files are prefixed by + this name (e.g., sample_name.genes.results) + + BASIC OPTIONS + --paired-end + Input reads are paired-end reads. (Default: off) + + --no-qualities + Input reads do not contain quality scores. (Default: off) + + --strand-specific + The RNA-Seq protocol used to generate the reads is strand specific, + i.e., all (upstream) reads are derived from the forward strand. This + option is equivalent to --forward-prob=1.0. With this option set, if + RSEM runs the Bowtie/Bowtie 2 aligner, the '--norc' Bowtie/Bowtie 2 + option will be used, which disables alignment to the reverse strand + of transcripts. (Default: off) + + --bowtie2 + Use Bowtie 2 instead of Bowtie to align reads. Since currently RSEM + does not handle indel, local and discordant alignments, the Bowtie2 + parameters are set in a way to avoid those alignments. In + particular, we use options '--sensitive --dpad 0 --gbar 99999999 + --mp 1,1 --np 1 --score-min L,0,-0.1' by default. The last parameter + of '--score-min', '-0.1', is the negative of maximum mismatch rate. + This rate can be set by option '--bowtie2-mismatch-rate'. If reads + are paired-end, we additionally use options '--no-mixed' and + '--no-discordant'. (Default: off) + + --sam + Input file is in SAM format. (Default: off) + + --bam + Input file is in BAM format. (Default: off) + + -p/--num-threads + Number of threads to use. Both Bowtie/Bowtie2, expression estimation + and 'samtools sort' will use this many threads. (Default: 1) + + --no-bam-output + Do not output any BAM file. (Default: off) + + --output-genome-bam + Generate a BAM file, 'sample_name.genome.bam', with alignments + mapped to genomic coordinates and annotated with their posterior + probabilities. In addition, RSEM will call samtools (included in + RSEM package) to sort and index the bam file. + 'sample_name.genome.sorted.bam' and + 'sample_name.genome.sorted.bam.bai' will be generated. (Default: + off) + + --sampling-for-bam + When RSEM generates a BAM file, instead of outputing all alignments + a read has with their posterior probabilities, one alignment is + sampled according to the posterior probabilities. The sampling + procedure includes the alignment to the "noise" transcript, which + does not appear in the BAM file. Only the sampled alignment has a + weight of 1. All other alignments have weight 0. If the "noise" + transcript is sampled, all alignments appeared in the BAM file + should have weight 0. (Default: off) + + --seed + Set the seed for the random number generators used in calculating + posterior mean estimates and credibility intervals. The seed must be + a non-negative 32 bit interger. (Default: off) + + --calc-pme + Run RSEM's collapsed Gibbs sampler to calculate posterior mean + estimates. (Default: off) + + --calc-ci + Calculate 95% credibility intervals and posterior mean estimates. + The credibility level can be changed by setting + '--ci-credibility-level'. (Default: off) + + -q/--quiet + Suppress the output of logging information. (Default: off) + + -h/--help + Show help information. + + --version + Show version information. + + ADVANCED OPTIONS + --sam-header-info + RSEM reads header information from input by default. If this option + is on, header information is read from the specified file. For the + format of the file, please see SAM official website. (Default: "") + + --seed-length + Seed length used by the read aligner. Providing the correct value is + important for RSEM. If RSEM runs Bowtie, it uses this value for + Bowtie's seed length parameter. Any read with its or at least one of + its mates' (for paired-end reads) length less than this value will + be ignored. If the references are not added poly(A) tails, the + minimum allowed value is 5, otherwise, the minimum allowed value is + 25. Note that this script will only check if the value >= 5 and give + a warning message if the value < 25 but >= 5. (Default: 25) + + --tag + The name of the optional field used in the SAM input for identifying + a read with too many valid alignments. The field should have the + format :i:, where a bigger than 0 indicates + a read with too many alignments. (Default: "") + + --bowtie-path + The path to the Bowtie executables. (Default: the path to the Bowtie + executables is assumed to be in the user's PATH environment + variable) + + --bowtie-n + (Bowtie parameter) max # of mismatches in the seed. (Range: 0-3, + Default: 2) + + --bowtie-e + (Bowtie parameter) max sum of mismatch quality scores across the + alignment. (Default: 99999999) + + --bowtie-m + (Bowtie parameter) suppress all alignments for a read if > + valid alignments exist. (Default: 200) + + --bowtie-chunkmbs + (Bowtie parameter) memory allocated for best first alignment + calculation (Default: 0 - use Bowtie's default) + + --phred33-quals + Input quality scores are encoded as Phred+33. (Default: on) + + --phred64-quals + Input quality scores are encoded as Phred+64 (default for GA + Pipeline ver. >= 1.3). (Default: off) + + --solexa-quals + Input quality scores are solexa encoded (from GA Pipeline ver. < + 1.3). (Default: off) + + --bowtie2-path + (Bowtie 2 parameter) The path to the Bowtie 2 executables. (Default: + the path to the Bowtie 2 executables is assumed to be in the user's + PATH environment variable) + + --bowtie2-mismatch-rate + (Bowtie 2 parameter) The maximum mismatch rate allowed. (Default: + 0.1) + + --bowtie2-k + (Bowtie 2 parameter) Find up to alignments per read. (Default: + 200) + + --bowtie2-sensitivity-level + (Bowtie 2 parameter) Set Bowtie 2's preset options in --end-to-end + mode. This option controls how hard Bowtie 2 tries to find + alignments. must be one of "very_fast", "fast", "sensitive" + and "very_sensitive". The four candidates correspond to Bowtie 2's + "--very-fast", "--fast", "--sensitive" and "--very-sensitive" + options. (Default: "sensitive" - use Bowtie 2's default) + + --forward-prob + Probability of generating a read from the forward strand of a + transcript. Set to 1 for a strand-specific protocol where all + (upstream) reads are derived from the forward strand, 0 for a + strand-specific protocol where all (upstream) read are derived from + the reverse strand, or 0.5 for a non-strand-specific protocol. + (Default: 0.5) + + --fragment-length-min + Minimum read/insert length allowed. This is also the value for the + Bowtie/Bowtie2 -I option. (Default: 1) + + --fragment-length-max + Maximum read/insert length allowed. This is also the value for the + Bowtie/Bowtie 2 -X option. (Default: 1000) + + --fragment-length-mean + (single-end data only) The mean of the fragment length distribution, + which is assumed to be a Gaussian. (Default: -1, which disables use + of the fragment length distribution) + + --fragment-length-sd + (single-end data only) The standard deviation of the fragment length + distribution, which is assumed to be a Gaussian. (Default: 0, which + assumes that all fragments are of the same length, given by the + rounded value of --fragment-length-mean) + + --estimate-rspd + Set this option if you want to estimate the read start position + distribution (RSPD) from data. Otherwise, RSEM will use a uniform + RSPD. (Default: off) + + --num-rspd-bins + Number of bins in the RSPD. Only relevant when '--estimate-rspd' is + specified. Use of the default setting is recommended. (Default: 20) + + --gibbs-burnin + The number of burn-in rounds for RSEM's Gibbs sampler. Each round + passes over the entire data set once. If RSEM can use multiple + threads, multiple Gibbs samplers will start at the same time and all + samplers share the same burn-in number. (Default: 200) + + --gibbs-number-of-samples + The total number of count vectors RSEM will collect from its Gibbs + samplers. (Default: 1000) + + --gibbs-sampling-gap + The number of rounds between two succinct count vectors RSEM + collects. If the count vector after round N is collected, the count + vector after round N + will also be collected. (Default: 1) + + --ci-credibility-level + The credibility level for credibility intervals. (Default: 0.95) + + --ci-memory + Maximum size (in memory, MB) of the auxiliary buffer used for + computing credibility intervals (CI). Set it larger for a faster CI + calculation. However, leaving 2 GB memory free for other usage is + recommended. (Default: 1024) + + --ci-number-of-samples-per-count-vector + The number of read generating probability vectors sampled per + sampled count vector. The crebility intervals are calculated by + first sampling P(C | D) and then sampling P(Theta | C) for each + sampled count vector. This option controls how many Theta vectors + are sampled per sampled count vector. (Default: 50) + + --samtools-sort-mem + Set the maximum memory per thread that can be used by 'samtools + sort'. represents the memory and accepts suffices 'K/M/G'. + RSEM will pass to the '-m' option of 'samtools sort'. + Please note that the default used here is different from the default + used by samtools. (Default: 1G) + + --keep-intermediate-files + Keep temporary files generated by RSEM. RSEM creates a temporary + directory, 'sample_name.temp', into which it puts all intermediate + output files. If this directory already exists, RSEM overwrites all + files generated by previous RSEM runs inside of it. By default, + after RSEM finishes, the temporary directory is deleted. Set this + option to prevent the deletion of this directory and the + intermediate files inside of it. (Default: off) + + --temporary-folder + Set where to put the temporary files generated by RSEM. If the + folder specified does not exist, RSEM will try to create it. + (Default: sample_name.temp) + + --time + Output time consumed by each step of RSEM to 'sample_name.time'. + (Default: off) + + DESCRIPTION + In its default mode, this program aligns input reads against a reference + transcriptome with Bowtie and calculates expression values using the + alignments. RSEM assumes the data are single-end reads with quality + scores, unless the '--paired-end' or '--no-qualities' options are + specified. Users may use an alternative aligner by specifying one of the + --sam and --bam options, and providing an alignment file in the + specified format. However, users should make sure that they align + against the indices generated by 'rsem-prepare-reference' and the + alignment file satisfies the requirements mentioned in ARGUMENTS + section. + + One simple way to make the alignment file satisfying RSEM's requirements + (assuming the aligner used put mates in a paired-end read adjacent) is + to use 'convert-sam-for-rsem' script. This script only accept SAM format + files as input. If a BAM format file is obtained, please use samtools to + convert it to a SAM file first. For example, if '/ref/mouse_125' is the + 'reference_name' and the SAM file is named 'input.sam', you can run the + following command: + + convert-sam-for-rsem /ref/mouse_125 input.sam -o input_for_rsem.sam + + For details, please refer to 'convert-sam-for-rsem's documentation page. + + The SAM/BAM format RSEM uses is v1.4. However, it is compatible with old + SAM/BAM format. However, RSEM cannot recognize 0x100 in the FLAG field. + In addition, RSEM requires SEQ and QUAL are not '*'. + + The user must run 'rsem-prepare-reference' with the appropriate + reference before using this program. + + For single-end data, it is strongly recommended that the user provide + the fragment length distribution parameters (--fragment-length-mean and + --fragment-length-sd). For paired-end data, RSEM will automatically + learn a fragment length distribution from the data. + + Please note that some of the default values for the Bowtie parameters + are not the same as those defined for Bowtie itself. + + The temporary directory and all intermediate files will be removed when + RSEM finishes unless '--keep-intermediate-files' is specified. + + With the '--calc-pme' option, posterior mean estimates will be + calculated in addition to maximum likelihood estimates. + + With the '--calc-ci' option, 95% credibility intervals and posterior + mean estimates will be calculated in addition to maximum likelihood + estimates. + + OUTPUT + sample_name.isoforms.results + File containing isoform level expression estimates. The first line + contains column names separated by the tab character. The format of + each line in the rest of this file is: + + transcript_id gene_id length effective_length expected_count TPM + FPKM IsoPct [posterior_mean_count + posterior_standard_deviation_of_count pme_TPM pme_FPKM + IsoPct_from_pme_TPM TPM_ci_lower_bound TPM_ci_upper_bound + FPKM_ci_lower_bound FPKM_ci_upper_bound] + + Fields are separated by the tab character. Fields within "[]" are + optional. They will not be presented if neither '--calc-pme' nor + '--calc-ci' is set. + + 'transcript_id' is the transcript name of this transcript. 'gene_id' + is the gene name of the gene which this transcript belongs to + (denote this gene as its parent gene). If no gene information is + provided, 'gene_id' and 'transcript_id' are the same. + + 'length' is this transcript's sequence length (poly(A) tail is not + counted). 'effective_length' counts only the positions that can + generate a valid fragment. If no poly(A) tail is added, + 'effective_length' is equal to transcript length - mean fragment + length + 1. If one transcript's effective length is less than 1, + this transcript's both effective length and abundance estimates are + set to 0. + + 'expected_count' is the sum of the posterior probability of each + read comes from this transcript over all reads. Because 1) each read + aligning to this transcript has a probability of being generated + from background noise; 2) RSEM may filter some alignable low quality + reads, the sum of expected counts for all transcript are generally + less than the total number of reads aligned. + + 'TPM' stands for Transcripts Per Million. It is a relative measure + of transcript abundance. The sum of all transcripts' TPM is 1 + million. 'FPKM' stands for Fragments Per Kilobase of transcript per + Million mapped reads. It is another relative measure of transcript + abundance. If we define l_bar be the mean transcript length in a + sample, which can be calculated as + + l_bar = \sum_i TPM_i / 10^6 * effective_length_i (i goes through + every transcript), + + the following equation is hold: + + FPKM_i = 10^3 / l_bar * TPM_i. + + We can see that the sum of FPKM is not a constant across samples. + + 'IsoPct' stands for isoform percentage. It is the percentage of this + transcript's abandunce over its parent gene's abandunce. If its + parent gene has only one isoform or the gene information is not + provided, this field will be set to 100. + + 'posterior_mean_count', 'pme_TPM', 'pme_FPKM' are posterior mean + estimates calculated by RSEM's Gibbs sampler. + 'posterior_standard_deviation_of_count' is the posterior standard + deviation of counts. 'IsoPct_from_pme_TPM' is the isoform percentage + calculated from 'pme_TPM' values. + + 'TPM_ci_lower_bound', 'TPM_ci_upper_bound', 'FPKM_ci_lower_bound' + and 'FPKM_ci_upper_bound' are lower(l) and upper(u) bounds of 95% + credibility intervals for TPM and FPKM values. The bounds are + inclusive (i.e. [l, u]). + + sample_name.genes.results + File containing gene level expression estimates. The first line + contains column names separated by the tab character. The format of + each line in the rest of this file is: + + gene_id transcript_id(s) length effective_length expected_count TPM + FPKM [posterior_mean_count posterior_standard_deviation_of_count + pme_TPM pme_FPKM TPM_ci_lower_bound TPM_ci_upper_bound + FPKM_ci_lower_bound FPKM_ci_upper_bound] + + Fields are separated by the tab character. Fields within "[]" are + optional. They will not be presented if neither '--calc-pme' nor + '--calc-ci' is set. + + 'transcript_id(s)' is a comma-separated list of transcript_ids + belonging to this gene. If no gene information is provided, + 'gene_id' and 'transcript_id(s)' are identical (the + 'transcript_id'). + + A gene's 'length' and 'effective_length' are defined as the weighted + average of its transcripts' lengths and effective lengths (weighted + by 'IsoPct'). A gene's abundance estimates are just the sum of its + transcripts' abundance estimates. + + sample_name.alleles.results + Only generated when the RSEM references are built with + allele-specific transcripts. + + This file contains allele level expression estimates for + allele-specific expression calculation. The first line contains + column names separated by the tab character. The format of each line + in the rest of this file is: + + allele_id transcript_id gene_id length effective_length + expected_count TPM FPKM AlleleIsoPct AlleleGenePct + [posterior_mean_count posterior_standard_deviation_of_count pme_TPM + pme_FPKM AlleleIsoPct_from_pme_TPM AlleleGenePct_from_pme_TPM + TPM_ci_lower_bound TPM_ci_upper_bound FPKM_ci_lower_bound + FPKM_ci_upper_bound] + + Fields are separated by the tab character. Fields within "[]" are + optional. They will not be presented if neither '--calc-pme' nor + '--calc-ci' is set. + + 'allele_id' is the allele-specific name of this allele-specific + transcript. + + 'AlleleIsoPct' stands for allele-specific percentage on isoform + level. It is the percentage of this allele-specific transcript's + abundance over its parent transcript's abundance. If its parent + transcript has only one allele variant form, this field will be set + to 100. + + 'AlleleGenePct' stands for allele-specific percentage on gene level. + It is the percentage of this allele-specific transcript's abundance + over its parent gene's abundance. + + 'AlleleIsoPct_from_pme_TPM' and 'AlleleGenePct_from_pme_TPM' have + similar meanings. They are calculated based on posterior mean + estimates. + + Please note that if this file is present, the fields 'length' and + 'effective_length' in 'sample_name.isoforms.results' should be + interpreted similarly as the corresponding definitions in + 'sample_name.genes.results'. + + sample_name.transcript.bam, sample_name.transcript.sorted.bam and + sample_name.transcript.sorted.bam.bai + Only generated when --no-bam-output is not specified. + + 'sample_name.transcript.bam' is a BAM-formatted file of read + alignments in transcript coordinates. The MAPQ field of each + alignment is set to min(100, floor(-10 * log10(1.0 - w) + 0.5)), + where w is the posterior probability of that alignment being the + true mapping of a read. In addition, RSEM pads a new tag ZW:f:value, + where value is a single precision floating number representing the + posterior probability. Because this file contains all alignment + lines produced by bowtie or user-specified aligners, it can also be + used as a replacement of the aligner generated BAM/SAM file. For + paired-end reads, if one mate has alignments but the other does not, + this file marks the alignable mate as "unmappable" (flag bit 0x4) + and appends an optional field "Z0:A:!". + + 'sample_name.transcript.sorted.bam' and + 'sample_name.transcript.sorted.bam.bai' are the sorted BAM file and + indices generated by samtools (included in RSEM package). + + sample_name.genome.bam, sample_name.genome.sorted.bam and + sample_name.genome.sorted.bam.bai + Only generated when --no-bam-output is not specified and + --output-genome-bam is specified. + + 'sample_name.genome.bam' is a BAM-formatted file of read alignments + in genomic coordinates. Alignments of reads that have identical + genomic coordinates (i.e., alignments to different isoforms that + share the same genomic region) are collapsed into one alignment. The + MAPQ field of each alignment is set to min(100, floor(-10 * + log10(1.0 - w) + 0.5)), where w is the posterior probability of that + alignment being the true mapping of a read. In addition, RSEM pads a + new tag ZW:f:value, where value is a single precision floating + number representing the posterior probability. If an alignment is + spliced, a XS:A:value tag is also added, where value is either '+' + or '-' indicating the strand of the transcript it aligns to. + + 'sample_name.genome.sorted.bam' and + 'sample_name.genome.sorted.bam.bai' are the sorted BAM file and + indices generated by samtools (included in RSEM package). + + sample_name.time + Only generated when --time is specified. + + It contains time (in seconds) consumed by aligning reads, estimating + expression levels and calculating credibility intervals. + + sample_name.stat + This is a folder instead of a file. All model related statistics are + stored in this folder. Use 'rsem-plot-model' can generate plots + using this folder. + + 'sample_name.stat/sample_name.cnt' contains alignment statistics. + The format and meanings of each field are described in + 'cnt_file_description.txt' under RSEM directory. + + 'sample_name.stat/sample_name.model' stores RNA-Seq model parameters + learned from the data. The format and meanings of each filed of this + file are described in 'model_file_description.txt' under RSEM + directory. + + EXAMPLES + Assume the path to the bowtie executables is in the user's PATH + environment variable. Reference files are under '/ref' with name + 'mouse_125'. + + 1) '/data/mmliver.fq', single-end reads with quality scores. Quality + scores are encoded as for 'GA pipeline version >= 1.3'. We want to use 8 + threads and generate a genome BAM file: + + rsem-calculate-expression --phred64-quals \ + -p 8 \ + --output-genome-bam \ + /data/mmliver.fq \ + /ref/mouse_125 \ + mmliver_single_quals + + 2) '/data/mmliver_1.fq' and '/data/mmliver_2.fq', paired-end reads with + quality scores. Quality scores are in SANGER format. We want to use 8 + threads and do not generate a genome BAM file: + + rsem-calculate-expression -p 8 \ + --paired-end \ + /data/mmliver_1.fq \ + /data/mmliver_2.fq \ + /ref/mouse_125 \ + mmliver_paired_end_quals + + 3) '/data/mmliver.fa', single-end reads without quality scores. We want + to use 8 threads: + + rsem-calculate-expression -p 8 \ + --no-qualities \ + /data/mmliver.fa \ + /ref/mouse_125 \ + mmliver_single_without_quals + + 4) Data are the same as 1). This time we assume the bowtie executables + are under '/sw/bowtie'. We want to take a fragment length distribution + into consideration. We set the fragment length mean to 150 and the + standard deviation to 35. In addition to a BAM file, we also want to + generate credibility intervals. We allow RSEM to use 1GB of memory for + CI calculation: + + rsem-calculate-expression --bowtie-path /sw/bowtie \ + --phred64-quals \ + --fragment-length-mean 150.0 \ + --fragment-length-sd 35.0 \ + -p 8 \ + --output-genome-bam \ + --calc-ci \ + --ci-memory 1024 \ + /data/mmliver.fq \ + /ref/mouse_125 \ + mmliver_single_quals + + 5) '/data/mmliver_paired_end_quals.bam', paired-end reads with quality + scores. We want to use 8 threads: + + rsem-calculate-expression --paired-end \ + --bam \ + -p 8 \ + /data/mmliver_paired_end_quals.bam \ + /ref/mouse_125 \ + mmliver_paired_end_quals + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpqb2aj7b1/3875e02b-ab2a-4776-a8d4-0fe0715cf84b:/data', '--log-driver=none', 'aarjunrao/rsem:1.2.20', '--paired-end', '-p', '80.0', '--bam', '/data/star_transcriptome.bam', '--no-bam-output', '/data/rsem_indexes/hg38', 'rsem']' returned non-zero exit status 2. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/expression_profiling/rsem.py", line 88, in run_rsem + dockerhub=univ_options['dockerhub'], tool_version=rsem_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (2)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpqb2aj7b1/3875e02b-ab2a-4776-a8d4-0fe0715cf84b:/data --log-driver=none aarjunrao/rsem:1.2.20 --paired-end -p 80.0 --bam /data/star_transcriptome.bam --no-bam-output /data/rsem_indexes/hg38 rsem" + [2020-11-17T11:52:04-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-17T11:52:05-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with ID kind-run_rsem/instance-mzzok2e3 to 1 +[2020-11-17T11:52:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with job batch system ID: 394 and cores: 80, disk: 5.0 G, and memory: 2.0 G +[2020-11-17T11:52:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpstl788nb/worker_log.txt +[2020-11-17T11:53:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:31: Filtered SomaticSniper for TEST successfully +[2020-11-17T11:53:32-0800] [MainThread] [I] [toil.leader] Job ended: 'filter_somaticsniper' kind-filter_somaticsniper/instance-27dvr2ro +[2020-11-17T11:53:32-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-kov87f82 with job batch system ID: 395 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:53:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphf8z7bhn/worker_log.txt +[2020-11-17T11:53:33-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-kov87f82 +[2020-11-17T11:53:33-0800] [MainThread] [I] [toil.leader] Issued job 'unmerge' kind-unmerge/instance-8v42p02i with job batch system ID: 396 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-17T11:53:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp1olum1li/worker_log.txt +[2020-11-17T11:53:35-0800] [MainThread] [I] [toil.leader] Job ended: 'unmerge' kind-unmerge/instance-8v42p02i +[2020-11-17T11:53:35-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-kov87f82 with job batch system ID: 397 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-dd0xa2mi/chr4.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-tjenbom0/chr6.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-khaiuiik/chr7.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-e3oyc_mg/chr8.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-v2tzcg8t/chr15.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-41yrx4bq/chr16.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-87lvo5a1/chrM.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-ue01fauq/chr2.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-olote6ss/chr20.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-3jhpe732/chr19.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-f59bwbgj/chr17.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-7wq4f5i6/chr12.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-ucx4wgli/chr3.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-r20clu4p/chrX.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-zcmnnhiz/chr13.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-g9pqe_pm/chrY.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-tl91acp7/chr14.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-heltayq3/chr1.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-2v95fqwr/chr5.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-axl7dyhp/chr21.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-d1bk19nn/chr18.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-f6qtx7g6/chr11.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-25e7vnbp/chr10.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-engistli/chr22.vcf to output location +[2020-11-17T11:53:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:53:35: Exporting files/for-job/kind-unmerge/instance-8v42p02i/file-j4tgsx11/chr9.vcf to output location +[2020-11-17T11:53:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpm41ra72b/worker_log.txt +[2020-11-17T11:53:36-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-kov87f82 +[2020-11-17T11:53:36-0800] [MainThread] [I] [toil.leader] Issued job 'filter_somaticsniper' kind-EncapsulatedJob/instance-to8zefli with job batch system ID: 398 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:53:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3083ewmw/worker_log.txt +[2020-11-17T11:53:36-0800] [MainThread] [I] [toil.leader] Job ended: 'filter_somaticsniper' kind-EncapsulatedJob/instance-to8zefli +[2020-11-17T11:53:36-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-r9pjl3mp with job batch system ID: 399 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:53:36-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-nxi6lpyj with job batch system ID: 400 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:53:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmps7qwjfc9/worker_log.txt +[2020-11-17T11:53:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp2yrtq7gt/worker_log.txt +[2020-11-17T11:53:37-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-r9pjl3mp +[2020-11-17T11:53:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_pileup' kind-EncapsulatedJob/instance-t0hrdp2k with job batch system ID: 401 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:53:37-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-nxi6lpyj +[2020-11-17T11:53:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_somaticsniper_full' kind-EncapsulatedJob/instance-wuf__ewu with job batch system ID: 402 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:53:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp7owklh2e/worker_log.txt +[2020-11-17T11:53:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfbs_1p77/worker_log.txt +[2020-11-17T11:53:38-0800] [MainThread] [I] [toil.leader] Job ended: 'run_pileup' kind-EncapsulatedJob/instance-t0hrdp2k +[2020-11-17T11:53:38-0800] [MainThread] [I] [toil.leader] Job ended: 'run_somaticsniper_full' kind-EncapsulatedJob/instance-wuf__ewu +[2020-11-17T11:53:38-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-r6f718fs with job batch system ID: 403 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:53:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0n_l4heg/worker_log.txt +[2020-11-17T11:53:38-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-r6f718fs +[2020-11-17T11:54:08-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:54:08: Ran radia on TEST:chrM successfully +[2020-11-17T11:54:09-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-l2grlbre +[2020-11-17T11:54:09-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-19huvccq with job batch system ID: 404 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:54:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfhopb_qj/worker_log.txt +[2020-11-17T11:54:10-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-19huvccq +[2020-11-17T11:54:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-ihh5y23p with job batch system ID: 405 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:54:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp09q6rsfh/worker_log.txt +[2020-11-17T11:55:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:55:43: Ran radia on TEST:chrY successfully +[2020-11-17T11:55:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-9_85_oup +[2020-11-17T11:55:44-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-3_nykzg2 with job batch system ID: 406 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpbfcsy650/worker_log.txt +[2020-11-17T11:55:45-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-3_nykzg2 +[2020-11-17T11:55:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-2df53bpj with job batch system ID: 407 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T11:55:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpp6cb9kdr/worker_log.txt +[2020-11-17T11:56:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:56:46: Exporting files/for-job/kind-run_filter_radia/instance-2df53bpj/file-0szf86aq/chrY.vcf to output location +[2020-11-17T11:56:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:56:46: Ran filter-radia on TEST:chrY successfully +[2020-11-17T11:56:46-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:56:46: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-3_nykzg2/cleanup/file-vc1wogy6/stream used 123.23% (6.5 GB [7020261376B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T11:56:47-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-2df53bpj +[2020-11-17T11:56:47-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-lidfugyt with job batch system ID: 408 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:56:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpqpkl_433/worker_log.txt +[2020-11-17T11:56:48-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-lidfugyt +[2020-11-17T11:56:48-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-3_nykzg2 with job batch system ID: 409 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:56:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmplet8bhtg/worker_log.txt +[2020-11-17T11:56:49-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-3_nykzg2 +[2020-11-17T11:56:49-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-l8a5rrhr with job batch system ID: 410 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:56:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmplzw6is7_/worker_log.txt +[2020-11-17T11:56:49-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-l8a5rrhr +[2020-11-17T11:57:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:57:28: Exporting files/for-job/kind-run_filter_radia/instance-ihh5y23p/file-kbzme8od/chrM.vcf to output location +[2020-11-17T11:57:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:57:28: Ran filter-radia on TEST:chrM successfully +[2020-11-17T11:57:28-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 11:57:28: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-19huvccq/cleanup/file-evbcs1oh/stream used 123.22% (6.5 GB [7019679744B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T11:57:28-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-ihh5y23p +[2020-11-17T11:57:28-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-sz21ypu1 with job batch system ID: 411 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:57:29-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpov0f3kix/worker_log.txt +[2020-11-17T11:57:29-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-sz21ypu1 +[2020-11-17T11:57:29-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-19huvccq with job batch system ID: 412 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T11:57:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpy9s4apw5/worker_log.txt +[2020-11-17T11:57:30-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-19huvccq +[2020-11-17T11:57:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-8af2a19g with job batch system ID: 413 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T11:57:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp414exast/worker_log.txt +[2020-11-17T11:57:30-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-8af2a19g +[2020-11-17T11:57:47-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_fusion file:/scratch/drkthomp/jobStore kind-run_fusion/instance-g9_082sr. +[2020-11-17T11:57:47-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-17T11:57:47-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-17T11:57:47-0800] [MainThread] [W] [toil.leader] Log from job kind-run_fusion/instance-g9_082sr follows: +=========> + [2020-11-17T11:52:06-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T11:52:06-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + 1.0.0: Pulling from aarjunrao/star-fusion + 943c334059c7: Already exists + a1acf99303d2: Already exists + 27616aacb7b3: Already exists + 35d12cd1c9fc: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + a27d4604b3b3: Already exists + b23184322fd8: Already exists + 45490e2c3ebf: Already exists + a3ed95caeb02: Already exists + 07bc45a5d3f9: Already exists + a3ed95caeb02: Already exists + 43bc14d13ff5: Already exists + ee9cc1e586c2: Already exists + 8502c8e3592c: Already exists + bbe2bace9f10: Already exists + 923fd5df245d: Already exists + 73894d39f68a: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + da70bf223bc8: Pulling fs layer + 5582ae3f96b5: Pulling fs layer + 8b5f4751bf65: Pulling fs layer + 0babb5b32a0f: Pulling fs layer + 32114e3e6039: Pulling fs layer + b53bb31848a7: Pulling fs layer + 32114e3e6039: Waiting + b53bb31848a7: Waiting + 0babb5b32a0f: Waiting + da70bf223bc8: Download complete + da70bf223bc8: Pull complete + 8b5f4751bf65: Verifying Checksum + 8b5f4751bf65: Download complete + 5582ae3f96b5: Verifying Checksum + 5582ae3f96b5: Download complete + 5582ae3f96b5: Pull complete + 0babb5b32a0f: Verifying Checksum + 0babb5b32a0f: Download complete + 8b5f4751bf65: Pull complete + b53bb31848a7: Download complete + 32114e3e6039: Verifying Checksum + 32114e3e6039: Download complete + 0babb5b32a0f: Pull complete + 32114e3e6039: Pull complete + b53bb31848a7: Pull complete + Digest: sha256:58a84d79b7b86f976029310e3945b14d887838fb603890d3991acd5e48ad526a + Status: Downloaded newer image for aarjunrao/star-fusion:1.0.0 + Error, don't understand arguments: [--CPU 80.0] at /opt/STAR-Fusion-v1.0.0/STAR-Fusion line 167. + [2020-11-17T11:57:41-0800] [MainThread] [W] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-v67rsjos/cleanup/file-d_nkrp7o/stream used 111.60% (59.6 GB [64023433216B] used, 53.4 GB [57369019908B] requested) at the end of its run. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpstl788nb/f30df2db-fbc5-4ce4-aac1-988269daf230/t84qq90gv:/data', '--log-driver=none', 'aarjunrao/star-fusion:1.0.0', '--chimeric_junction', '/data/STAR.junction', '--output_dir', '/data/fusion-output', '--genome_lib_dir', 'star_with_fusion_100bp_readlen_index', '--CPU', '80.0']' returned non-zero exit status 255. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/fusion.py", line 118, in run_fusion + tool_version=star_fusion_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (255)for command "docker run --rm=true -v /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpstl788nb/f30df2db-fbc5-4ce4-aac1-988269daf230/t84qq90gv:/data --log-driver=none aarjunrao/star-fusion:1.0.0 --chimeric_junction /data/STAR.junction --output_dir /data/fusion-output --genome_lib_dir star_with_fusion_100bp_readlen_index --CPU 80.0" + [2020-11-17T11:57:41-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-17T11:57:47-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_fusion' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr to 1 +[2020-11-17T11:57:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_fusion' kind-run_fusion/instance-g9_082sr with job batch system ID: 414 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-17T11:57:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvcku4g7a/worker_log.txt +[2020-11-17T11:58:38-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_rsem file:/scratch/drkthomp/jobStore kind-run_rsem/instance-mzzok2e3. +[2020-11-17T11:58:38-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-17T11:58:38-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-17T11:58:38-0800] [MainThread] [W] [toil.leader] Log from job kind-run_rsem/instance-mzzok2e3 follows: +=========> + [2020-11-17T11:57:47-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T11:57:47-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Value "80.0" invalid for option p (number expected) + NAME + rsem-calculate-expression + + SYNOPSIS + rsem-calculate-expression [options] upstream_read_file(s) reference_name sample_name + rsem-calculate-expression [options] --paired-end upstream_read_file(s) downstream_read_file(s) reference_name sample_name + rsem-calculate-expression [options] --sam/--bam [--paired-end] input reference_name sample_name + + ARGUMENTS + upstream_read_files(s) + Comma-separated list of files containing single-end reads or + upstream reads for paired-end data. By default, these files are + assumed to be in FASTQ format. If the --no-qualities option is + specified, then FASTA format is expected. + + downstream_read_file(s) + Comma-separated list of files containing downstream reads which are + paired with the upstream reads. By default, these files are assumed + to be in FASTQ format. If the --no-qualities option is specified, + then FASTA format is expected. + + input + SAM/BAM formatted input file. If "-" is specified for the filename, + SAM/BAM input is instead assumed to come from standard input. RSEM + requires all alignments of the same read group together. For + paired-end reads, RSEM also requires the two mates of any alignment + be adjacent. See Description section for how to make input file obey + RSEM's requirements. + + reference_name + The name of the reference used. The user must have run + 'rsem-prepare-reference' with this reference_name before running + this program. + + sample_name + The name of the sample analyzed. All output files are prefixed by + this name (e.g., sample_name.genes.results) + + BASIC OPTIONS + --paired-end + Input reads are paired-end reads. (Default: off) + + --no-qualities + Input reads do not contain quality scores. (Default: off) + + --strand-specific + The RNA-Seq protocol used to generate the reads is strand specific, + i.e., all (upstream) reads are derived from the forward strand. This + option is equivalent to --forward-prob=1.0. With this option set, if + RSEM runs the Bowtie/Bowtie 2 aligner, the '--norc' Bowtie/Bowtie 2 + option will be used, which disables alignment to the reverse strand + of transcripts. (Default: off) + + --bowtie2 + Use Bowtie 2 instead of Bowtie to align reads. Since currently RSEM + does not handle indel, local and discordant alignments, the Bowtie2 + parameters are set in a way to avoid those alignments. In + particular, we use options '--sensitive --dpad 0 --gbar 99999999 + --mp 1,1 --np 1 --score-min L,0,-0.1' by default. The last parameter + of '--score-min', '-0.1', is the negative of maximum mismatch rate. + This rate can be set by option '--bowtie2-mismatch-rate'. If reads + are paired-end, we additionally use options '--no-mixed' and + '--no-discordant'. (Default: off) + + --sam + Input file is in SAM format. (Default: off) + + --bam + Input file is in BAM format. (Default: off) + + -p/--num-threads + Number of threads to use. Both Bowtie/Bowtie2, expression estimation + and 'samtools sort' will use this many threads. (Default: 1) + + --no-bam-output + Do not output any BAM file. (Default: off) + + --output-genome-bam + Generate a BAM file, 'sample_name.genome.bam', with alignments + mapped to genomic coordinates and annotated with their posterior + probabilities. In addition, RSEM will call samtools (included in + RSEM package) to sort and index the bam file. + 'sample_name.genome.sorted.bam' and + 'sample_name.genome.sorted.bam.bai' will be generated. (Default: + off) + + --sampling-for-bam + When RSEM generates a BAM file, instead of outputing all alignments + a read has with their posterior probabilities, one alignment is + sampled according to the posterior probabilities. The sampling + procedure includes the alignment to the "noise" transcript, which + does not appear in the BAM file. Only the sampled alignment has a + weight of 1. All other alignments have weight 0. If the "noise" + transcript is sampled, all alignments appeared in the BAM file + should have weight 0. (Default: off) + + --seed + Set the seed for the random number generators used in calculating + posterior mean estimates and credibility intervals. The seed must be + a non-negative 32 bit interger. (Default: off) + + --calc-pme + Run RSEM's collapsed Gibbs sampler to calculate posterior mean + estimates. (Default: off) + + --calc-ci + Calculate 95% credibility intervals and posterior mean estimates. + The credibility level can be changed by setting + '--ci-credibility-level'. (Default: off) + + -q/--quiet + Suppress the output of logging information. (Default: off) + + -h/--help + Show help information. + + --version + Show version information. + + ADVANCED OPTIONS + --sam-header-info + RSEM reads header information from input by default. If this option + is on, header information is read from the specified file. For the + format of the file, please see SAM official website. (Default: "") + + --seed-length + Seed length used by the read aligner. Providing the correct value is + important for RSEM. If RSEM runs Bowtie, it uses this value for + Bowtie's seed length parameter. Any read with its or at least one of + its mates' (for paired-end reads) length less than this value will + be ignored. If the references are not added poly(A) tails, the + minimum allowed value is 5, otherwise, the minimum allowed value is + 25. Note that this script will only check if the value >= 5 and give + a warning message if the value < 25 but >= 5. (Default: 25) + + --tag + The name of the optional field used in the SAM input for identifying + a read with too many valid alignments. The field should have the + format :i:, where a bigger than 0 indicates + a read with too many alignments. (Default: "") + + --bowtie-path + The path to the Bowtie executables. (Default: the path to the Bowtie + executables is assumed to be in the user's PATH environment + variable) + + --bowtie-n + (Bowtie parameter) max # of mismatches in the seed. (Range: 0-3, + Default: 2) + + --bowtie-e + (Bowtie parameter) max sum of mismatch quality scores across the + alignment. (Default: 99999999) + + --bowtie-m + (Bowtie parameter) suppress all alignments for a read if > + valid alignments exist. (Default: 200) + + --bowtie-chunkmbs + (Bowtie parameter) memory allocated for best first alignment + calculation (Default: 0 - use Bowtie's default) + + --phred33-quals + Input quality scores are encoded as Phred+33. (Default: on) + + --phred64-quals + Input quality scores are encoded as Phred+64 (default for GA + Pipeline ver. >= 1.3). (Default: off) + + --solexa-quals + Input quality scores are solexa encoded (from GA Pipeline ver. < + 1.3). (Default: off) + + --bowtie2-path + (Bowtie 2 parameter) The path to the Bowtie 2 executables. (Default: + the path to the Bowtie 2 executables is assumed to be in the user's + PATH environment variable) + + --bowtie2-mismatch-rate + (Bowtie 2 parameter) The maximum mismatch rate allowed. (Default: + 0.1) + + --bowtie2-k + (Bowtie 2 parameter) Find up to alignments per read. (Default: + 200) + + --bowtie2-sensitivity-level + (Bowtie 2 parameter) Set Bowtie 2's preset options in --end-to-end + mode. This option controls how hard Bowtie 2 tries to find + alignments. must be one of "very_fast", "fast", "sensitive" + and "very_sensitive". The four candidates correspond to Bowtie 2's + "--very-fast", "--fast", "--sensitive" and "--very-sensitive" + options. (Default: "sensitive" - use Bowtie 2's default) + + --forward-prob + Probability of generating a read from the forward strand of a + transcript. Set to 1 for a strand-specific protocol where all + (upstream) reads are derived from the forward strand, 0 for a + strand-specific protocol where all (upstream) read are derived from + the reverse strand, or 0.5 for a non-strand-specific protocol. + (Default: 0.5) + + --fragment-length-min + Minimum read/insert length allowed. This is also the value for the + Bowtie/Bowtie2 -I option. (Default: 1) + + --fragment-length-max + Maximum read/insert length allowed. This is also the value for the + Bowtie/Bowtie 2 -X option. (Default: 1000) + + --fragment-length-mean + (single-end data only) The mean of the fragment length distribution, + which is assumed to be a Gaussian. (Default: -1, which disables use + of the fragment length distribution) + + --fragment-length-sd + (single-end data only) The standard deviation of the fragment length + distribution, which is assumed to be a Gaussian. (Default: 0, which + assumes that all fragments are of the same length, given by the + rounded value of --fragment-length-mean) + + --estimate-rspd + Set this option if you want to estimate the read start position + distribution (RSPD) from data. Otherwise, RSEM will use a uniform + RSPD. (Default: off) + + --num-rspd-bins + Number of bins in the RSPD. Only relevant when '--estimate-rspd' is + specified. Use of the default setting is recommended. (Default: 20) + + --gibbs-burnin + The number of burn-in rounds for RSEM's Gibbs sampler. Each round + passes over the entire data set once. If RSEM can use multiple + threads, multiple Gibbs samplers will start at the same time and all + samplers share the same burn-in number. (Default: 200) + + --gibbs-number-of-samples + The total number of count vectors RSEM will collect from its Gibbs + samplers. (Default: 1000) + + --gibbs-sampling-gap + The number of rounds between two succinct count vectors RSEM + collects. If the count vector after round N is collected, the count + vector after round N + will also be collected. (Default: 1) + + --ci-credibility-level + The credibility level for credibility intervals. (Default: 0.95) + + --ci-memory + Maximum size (in memory, MB) of the auxiliary buffer used for + computing credibility intervals (CI). Set it larger for a faster CI + calculation. However, leaving 2 GB memory free for other usage is + recommended. (Default: 1024) + + --ci-number-of-samples-per-count-vector + The number of read generating probability vectors sampled per + sampled count vector. The crebility intervals are calculated by + first sampling P(C | D) and then sampling P(Theta | C) for each + sampled count vector. This option controls how many Theta vectors + are sampled per sampled count vector. (Default: 50) + + --samtools-sort-mem + Set the maximum memory per thread that can be used by 'samtools + sort'. represents the memory and accepts suffices 'K/M/G'. + RSEM will pass to the '-m' option of 'samtools sort'. + Please note that the default used here is different from the default + used by samtools. (Default: 1G) + + --keep-intermediate-files + Keep temporary files generated by RSEM. RSEM creates a temporary + directory, 'sample_name.temp', into which it puts all intermediate + output files. If this directory already exists, RSEM overwrites all + files generated by previous RSEM runs inside of it. By default, + after RSEM finishes, the temporary directory is deleted. Set this + option to prevent the deletion of this directory and the + intermediate files inside of it. (Default: off) + + --temporary-folder + Set where to put the temporary files generated by RSEM. If the + folder specified does not exist, RSEM will try to create it. + (Default: sample_name.temp) + + --time + Output time consumed by each step of RSEM to 'sample_name.time'. + (Default: off) + + DESCRIPTION + In its default mode, this program aligns input reads against a reference + transcriptome with Bowtie and calculates expression values using the + alignments. RSEM assumes the data are single-end reads with quality + scores, unless the '--paired-end' or '--no-qualities' options are + specified. Users may use an alternative aligner by specifying one of the + --sam and --bam options, and providing an alignment file in the + specified format. However, users should make sure that they align + against the indices generated by 'rsem-prepare-reference' and the + alignment file satisfies the requirements mentioned in ARGUMENTS + section. + + One simple way to make the alignment file satisfying RSEM's requirements + (assuming the aligner used put mates in a paired-end read adjacent) is + to use 'convert-sam-for-rsem' script. This script only accept SAM format + files as input. If a BAM format file is obtained, please use samtools to + convert it to a SAM file first. For example, if '/ref/mouse_125' is the + 'reference_name' and the SAM file is named 'input.sam', you can run the + following command: + + convert-sam-for-rsem /ref/mouse_125 input.sam -o input_for_rsem.sam + + For details, please refer to 'convert-sam-for-rsem's documentation page. + + The SAM/BAM format RSEM uses is v1.4. However, it is compatible with old + SAM/BAM format. However, RSEM cannot recognize 0x100 in the FLAG field. + In addition, RSEM requires SEQ and QUAL are not '*'. + + The user must run 'rsem-prepare-reference' with the appropriate + reference before using this program. + + For single-end data, it is strongly recommended that the user provide + the fragment length distribution parameters (--fragment-length-mean and + --fragment-length-sd). For paired-end data, RSEM will automatically + learn a fragment length distribution from the data. + + Please note that some of the default values for the Bowtie parameters + are not the same as those defined for Bowtie itself. + + The temporary directory and all intermediate files will be removed when + RSEM finishes unless '--keep-intermediate-files' is specified. + + With the '--calc-pme' option, posterior mean estimates will be + calculated in addition to maximum likelihood estimates. + + With the '--calc-ci' option, 95% credibility intervals and posterior + mean estimates will be calculated in addition to maximum likelihood + estimates. + + OUTPUT + sample_name.isoforms.results + File containing isoform level expression estimates. The first line + contains column names separated by the tab character. The format of + each line in the rest of this file is: + + transcript_id gene_id length effective_length expected_count TPM + FPKM IsoPct [posterior_mean_count + posterior_standard_deviation_of_count pme_TPM pme_FPKM + IsoPct_from_pme_TPM TPM_ci_lower_bound TPM_ci_upper_bound + FPKM_ci_lower_bound FPKM_ci_upper_bound] + + Fields are separated by the tab character. Fields within "[]" are + optional. They will not be presented if neither '--calc-pme' nor + '--calc-ci' is set. + + 'transcript_id' is the transcript name of this transcript. 'gene_id' + is the gene name of the gene which this transcript belongs to + (denote this gene as its parent gene). If no gene information is + provided, 'gene_id' and 'transcript_id' are the same. + + 'length' is this transcript's sequence length (poly(A) tail is not + counted). 'effective_length' counts only the positions that can + generate a valid fragment. If no poly(A) tail is added, + 'effective_length' is equal to transcript length - mean fragment + length + 1. If one transcript's effective length is less than 1, + this transcript's both effective length and abundance estimates are + set to 0. + + 'expected_count' is the sum of the posterior probability of each + read comes from this transcript over all reads. Because 1) each read + aligning to this transcript has a probability of being generated + from background noise; 2) RSEM may filter some alignable low quality + reads, the sum of expected counts for all transcript are generally + less than the total number of reads aligned. + + 'TPM' stands for Transcripts Per Million. It is a relative measure + of transcript abundance. The sum of all transcripts' TPM is 1 + million. 'FPKM' stands for Fragments Per Kilobase of transcript per + Million mapped reads. It is another relative measure of transcript + abundance. If we define l_bar be the mean transcript length in a + sample, which can be calculated as + + l_bar = \sum_i TPM_i / 10^6 * effective_length_i (i goes through + every transcript), + + the following equation is hold: + + FPKM_i = 10^3 / l_bar * TPM_i. + + We can see that the sum of FPKM is not a constant across samples. + + 'IsoPct' stands for isoform percentage. It is the percentage of this + transcript's abandunce over its parent gene's abandunce. If its + parent gene has only one isoform or the gene information is not + provided, this field will be set to 100. + + 'posterior_mean_count', 'pme_TPM', 'pme_FPKM' are posterior mean + estimates calculated by RSEM's Gibbs sampler. + 'posterior_standard_deviation_of_count' is the posterior standard + deviation of counts. 'IsoPct_from_pme_TPM' is the isoform percentage + calculated from 'pme_TPM' values. + + 'TPM_ci_lower_bound', 'TPM_ci_upper_bound', 'FPKM_ci_lower_bound' + and 'FPKM_ci_upper_bound' are lower(l) and upper(u) bounds of 95% + credibility intervals for TPM and FPKM values. The bounds are + inclusive (i.e. [l, u]). + + sample_name.genes.results + File containing gene level expression estimates. The first line + contains column names separated by the tab character. The format of + each line in the rest of this file is: + + gene_id transcript_id(s) length effective_length expected_count TPM + FPKM [posterior_mean_count posterior_standard_deviation_of_count + pme_TPM pme_FPKM TPM_ci_lower_bound TPM_ci_upper_bound + FPKM_ci_lower_bound FPKM_ci_upper_bound] + + Fields are separated by the tab character. Fields within "[]" are + optional. They will not be presented if neither '--calc-pme' nor + '--calc-ci' is set. + + 'transcript_id(s)' is a comma-separated list of transcript_ids + belonging to this gene. If no gene information is provided, + 'gene_id' and 'transcript_id(s)' are identical (the + 'transcript_id'). + + A gene's 'length' and 'effective_length' are defined as the weighted + average of its transcripts' lengths and effective lengths (weighted + by 'IsoPct'). A gene's abundance estimates are just the sum of its + transcripts' abundance estimates. + + sample_name.alleles.results + Only generated when the RSEM references are built with + allele-specific transcripts. + + This file contains allele level expression estimates for + allele-specific expression calculation. The first line contains + column names separated by the tab character. The format of each line + in the rest of this file is: + + allele_id transcript_id gene_id length effective_length + expected_count TPM FPKM AlleleIsoPct AlleleGenePct + [posterior_mean_count posterior_standard_deviation_of_count pme_TPM + pme_FPKM AlleleIsoPct_from_pme_TPM AlleleGenePct_from_pme_TPM + TPM_ci_lower_bound TPM_ci_upper_bound FPKM_ci_lower_bound + FPKM_ci_upper_bound] + + Fields are separated by the tab character. Fields within "[]" are + optional. They will not be presented if neither '--calc-pme' nor + '--calc-ci' is set. + + 'allele_id' is the allele-specific name of this allele-specific + transcript. + + 'AlleleIsoPct' stands for allele-specific percentage on isoform + level. It is the percentage of this allele-specific transcript's + abundance over its parent transcript's abundance. If its parent + transcript has only one allele variant form, this field will be set + to 100. + + 'AlleleGenePct' stands for allele-specific percentage on gene level. + It is the percentage of this allele-specific transcript's abundance + over its parent gene's abundance. + + 'AlleleIsoPct_from_pme_TPM' and 'AlleleGenePct_from_pme_TPM' have + similar meanings. They are calculated based on posterior mean + estimates. + + Please note that if this file is present, the fields 'length' and + 'effective_length' in 'sample_name.isoforms.results' should be + interpreted similarly as the corresponding definitions in + 'sample_name.genes.results'. + + sample_name.transcript.bam, sample_name.transcript.sorted.bam and + sample_name.transcript.sorted.bam.bai + Only generated when --no-bam-output is not specified. + + 'sample_name.transcript.bam' is a BAM-formatted file of read + alignments in transcript coordinates. The MAPQ field of each + alignment is set to min(100, floor(-10 * log10(1.0 - w) + 0.5)), + where w is the posterior probability of that alignment being the + true mapping of a read. In addition, RSEM pads a new tag ZW:f:value, + where value is a single precision floating number representing the + posterior probability. Because this file contains all alignment + lines produced by bowtie or user-specified aligners, it can also be + used as a replacement of the aligner generated BAM/SAM file. For + paired-end reads, if one mate has alignments but the other does not, + this file marks the alignable mate as "unmappable" (flag bit 0x4) + and appends an optional field "Z0:A:!". + + 'sample_name.transcript.sorted.bam' and + 'sample_name.transcript.sorted.bam.bai' are the sorted BAM file and + indices generated by samtools (included in RSEM package). + + sample_name.genome.bam, sample_name.genome.sorted.bam and + sample_name.genome.sorted.bam.bai + Only generated when --no-bam-output is not specified and + --output-genome-bam is specified. + + 'sample_name.genome.bam' is a BAM-formatted file of read alignments + in genomic coordinates. Alignments of reads that have identical + genomic coordinates (i.e., alignments to different isoforms that + share the same genomic region) are collapsed into one alignment. The + MAPQ field of each alignment is set to min(100, floor(-10 * + log10(1.0 - w) + 0.5)), where w is the posterior probability of that + alignment being the true mapping of a read. In addition, RSEM pads a + new tag ZW:f:value, where value is a single precision floating + number representing the posterior probability. If an alignment is + spliced, a XS:A:value tag is also added, where value is either '+' + or '-' indicating the strand of the transcript it aligns to. + + 'sample_name.genome.sorted.bam' and + 'sample_name.genome.sorted.bam.bai' are the sorted BAM file and + indices generated by samtools (included in RSEM package). + + sample_name.time + Only generated when --time is specified. + + It contains time (in seconds) consumed by aligning reads, estimating + expression levels and calculating credibility intervals. + + sample_name.stat + This is a folder instead of a file. All model related statistics are + stored in this folder. Use 'rsem-plot-model' can generate plots + using this folder. + + 'sample_name.stat/sample_name.cnt' contains alignment statistics. + The format and meanings of each field are described in + 'cnt_file_description.txt' under RSEM directory. + + 'sample_name.stat/sample_name.model' stores RNA-Seq model parameters + learned from the data. The format and meanings of each filed of this + file are described in 'model_file_description.txt' under RSEM + directory. + + EXAMPLES + Assume the path to the bowtie executables is in the user's PATH + environment variable. Reference files are under '/ref' with name + 'mouse_125'. + + 1) '/data/mmliver.fq', single-end reads with quality scores. Quality + scores are encoded as for 'GA pipeline version >= 1.3'. We want to use 8 + threads and generate a genome BAM file: + + rsem-calculate-expression --phred64-quals \ + -p 8 \ + --output-genome-bam \ + /data/mmliver.fq \ + /ref/mouse_125 \ + mmliver_single_quals + + 2) '/data/mmliver_1.fq' and '/data/mmliver_2.fq', paired-end reads with + quality scores. Quality scores are in SANGER format. We want to use 8 + threads and do not generate a genome BAM file: + + rsem-calculate-expression -p 8 \ + --paired-end \ + /data/mmliver_1.fq \ + /data/mmliver_2.fq \ + /ref/mouse_125 \ + mmliver_paired_end_quals + + 3) '/data/mmliver.fa', single-end reads without quality scores. We want + to use 8 threads: + + rsem-calculate-expression -p 8 \ + --no-qualities \ + /data/mmliver.fa \ + /ref/mouse_125 \ + mmliver_single_without_quals + + 4) Data are the same as 1). This time we assume the bowtie executables + are under '/sw/bowtie'. We want to take a fragment length distribution + into consideration. We set the fragment length mean to 150 and the + standard deviation to 35. In addition to a BAM file, we also want to + generate credibility intervals. We allow RSEM to use 1GB of memory for + CI calculation: + + rsem-calculate-expression --bowtie-path /sw/bowtie \ + --phred64-quals \ + --fragment-length-mean 150.0 \ + --fragment-length-sd 35.0 \ + -p 8 \ + --output-genome-bam \ + --calc-ci \ + --ci-memory 1024 \ + /data/mmliver.fq \ + /ref/mouse_125 \ + mmliver_single_quals + + 5) '/data/mmliver_paired_end_quals.bam', paired-end reads with quality + scores. We want to use 8 threads: + + rsem-calculate-expression --paired-end \ + --bam \ + -p 8 \ + /data/mmliver_paired_end_quals.bam \ + /ref/mouse_125 \ + mmliver_paired_end_quals + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvcku4g7a/010f0cce-4c29-4741-8bc0-4cf8309a14c1:/data', '--log-driver=none', 'aarjunrao/rsem:1.2.20', '--paired-end', '-p', '80.0', '--bam', '/data/star_transcriptome.bam', '--no-bam-output', '/data/rsem_indexes/hg38', 'rsem']' returned non-zero exit status 2. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/expression_profiling/rsem.py", line 88, in run_rsem + dockerhub=univ_options['dockerhub'], tool_version=rsem_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (2)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvcku4g7a/010f0cce-4c29-4741-8bc0-4cf8309a14c1:/data --log-driver=none aarjunrao/rsem:1.2.20 --paired-end -p 80.0 --bam /data/star_transcriptome.bam --no-bam-output /data/rsem_indexes/hg38 rsem" + [2020-11-17T11:58:37-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-17T11:58:38-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with ID kind-run_rsem/instance-mzzok2e3 to 0 +[2020-11-17T11:58:38-0800] [MainThread] [W] [toil.leader] Job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with ID kind-run_rsem/instance-mzzok2e3 is completely failed +[2020-11-17T11:58:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp70yydtrl/worker_log.txt +[2020-11-17T12:00:23-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:00:23: Ran radia on TEST:chr21 successfully +[2020-11-17T12:00:24-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-ltaggypj +[2020-11-17T12:00:24-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-6f876gws with job batch system ID: 415 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpzlnr6133/worker_log.txt +[2020-11-17T12:00:24-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-6f876gws +[2020-11-17T12:00:24-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-g4z059ke with job batch system ID: 416 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:00:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp2e0_gyeg/worker_log.txt +[2020-11-17T12:01:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:01:28: Exporting files/for-job/kind-run_filter_radia/instance-g4z059ke/file-6dkrs_j0/chr21.vcf to output location +[2020-11-17T12:01:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:01:28: Ran filter-radia on TEST:chr21 successfully +[2020-11-17T12:01:28-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:01:28: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-6f876gws/cleanup/file-gsfq0w0i/stream used 123.18% (6.5 GB [7017541632B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:01:29-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-g4z059ke +[2020-11-17T12:01:29-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-paaec2ea with job batch system ID: 417 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:01:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpffc69e1_/worker_log.txt +[2020-11-17T12:01:30-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-paaec2ea +[2020-11-17T12:01:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-6f876gws with job batch system ID: 418 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:01:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpyv5j9nre/worker_log.txt +[2020-11-17T12:01:31-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-6f876gws +[2020-11-17T12:01:31-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-4q5d86eb with job batch system ID: 419 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:01:31-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmplfh8e5vv/worker_log.txt +[2020-11-17T12:01:31-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-4q5d86eb +[2020-11-17T12:02:21-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:02:21: Ran radia on TEST:chr22 successfully +[2020-11-17T12:02:21-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-8y_bwf3d +[2020-11-17T12:02:21-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-x_f8itbu with job batch system ID: 420 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:02:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpef8w1d6m/worker_log.txt +[2020-11-17T12:02:22-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-x_f8itbu +[2020-11-17T12:02:22-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-0ywzsbm2 with job batch system ID: 421 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:02:23-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp5cr9ubzt/worker_log.txt +[2020-11-17T12:03:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:03:28: Exporting files/for-job/kind-run_filter_radia/instance-0ywzsbm2/file-lftfplsf/chr22.vcf to output location +[2020-11-17T12:03:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:03:28: Ran filter-radia on TEST:chr22 successfully +[2020-11-17T12:03:28-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:03:28: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-x_f8itbu/cleanup/file-5vmlk69x/stream used 123.19% (6.5 GB [7018127360B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:03:29-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-0ywzsbm2 +[2020-11-17T12:03:29-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-98isq9bh with job batch system ID: 422 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:03:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8pgm5m4l/worker_log.txt +[2020-11-17T12:03:35-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-98isq9bh +[2020-11-17T12:03:35-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-x_f8itbu with job batch system ID: 423 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:03:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp2xiactsx/worker_log.txt +[2020-11-17T12:03:35-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-x_f8itbu +[2020-11-17T12:03:35-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-o08r_0ko with job batch system ID: 424 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:03:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpua2k9m1a/worker_log.txt +[2020-11-17T12:03:36-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-o08r_0ko +[2020-11-17T12:04:21-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_fusion file:/scratch/drkthomp/jobStore kind-run_fusion/instance-g9_082sr. +[2020-11-17T12:04:21-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-17T12:04:21-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-17T12:04:21-0800] [MainThread] [W] [toil.leader] Log from job kind-run_fusion/instance-g9_082sr follows: +=========> + [2020-11-17T11:58:38-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T11:58:38-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Error, don't understand arguments: [--CPU 80.0] at /opt/STAR-Fusion-v1.0.0/STAR-Fusion line 167. + [2020-11-17T12:04:15-0800] [MainThread] [W] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-v67rsjos/cleanup/file-d_nkrp7o/stream used 111.60% (59.6 GB [64023433216B] used, 53.4 GB [57369019908B] requested) at the end of its run. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp70yydtrl/b4330403-dbac-473f-802d-d97e6cc860a2/tix_rseux:/data', '--log-driver=none', 'aarjunrao/star-fusion:1.0.0', '--chimeric_junction', '/data/STAR.junction', '--output_dir', '/data/fusion-output', '--genome_lib_dir', 'star_with_fusion_100bp_readlen_index', '--CPU', '80.0']' returned non-zero exit status 255. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/fusion.py", line 118, in run_fusion + tool_version=star_fusion_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (255)for command "docker run --rm=true -v /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp70yydtrl/b4330403-dbac-473f-802d-d97e6cc860a2/tix_rseux:/data --log-driver=none aarjunrao/star-fusion:1.0.0 --chimeric_junction /data/STAR.junction --output_dir /data/fusion-output --genome_lib_dir star_with_fusion_100bp_readlen_index --CPU 80.0" + [2020-11-17T12:04:15-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-17T12:04:21-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_fusion' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr to 0 +[2020-11-17T12:04:21-0800] [MainThread] [W] [toil.leader] Job 'run_fusion' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr is completely failed +[2020-11-17T12:04:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:04:45: Ran radia on TEST:chr19 successfully +[2020-11-17T12:04:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-s9p3b8oc +[2020-11-17T12:04:46-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-hf25q1_a with job batch system ID: 425 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:04:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmprv7jkcge/worker_log.txt +[2020-11-17T12:04:47-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-hf25q1_a +[2020-11-17T12:04:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-u52_qdkx with job batch system ID: 426 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:04:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpepotf6yl/worker_log.txt +[2020-11-17T12:05:52-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:05:52: Ran radia on TEST:chr20 successfully +[2020-11-17T12:05:52-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-aoi37xzh +[2020-11-17T12:05:52-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-defhtklq with job batch system ID: 427 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:05:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9s78b4g2/worker_log.txt +[2020-11-17T12:05:53-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-defhtklq +[2020-11-17T12:05:53-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-cnisdxvs with job batch system ID: 428 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:05:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmppfonwm_7/worker_log.txt +[2020-11-17T12:06:02-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:06:02: Exporting files/for-job/kind-run_filter_radia/instance-u52_qdkx/file-9enkk076/chr19.vcf to output location +[2020-11-17T12:06:02-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:06:02: Ran filter-radia on TEST:chr19 successfully +[2020-11-17T12:06:02-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:06:02: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-hf25q1_a/cleanup/file-g3hmuw1e/stream used 123.21% (6.5 GB [7019339776B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:06:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-u52_qdkx +[2020-11-17T12:06:03-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-vofa4ykc with job batch system ID: 429 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:06:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpwcu5bb64/worker_log.txt +[2020-11-17T12:06:03-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-vofa4ykc +[2020-11-17T12:06:03-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-hf25q1_a with job batch system ID: 430 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:06:04-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp2xbzl_y6/worker_log.txt +[2020-11-17T12:06:04-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-hf25q1_a +[2020-11-17T12:06:04-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-x7cm8te9 with job batch system ID: 431 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:06:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpmy5_2m5s/worker_log.txt +[2020-11-17T12:06:05-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-x7cm8te9 +[2020-11-17T12:07:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:07:03: Exporting files/for-job/kind-run_filter_radia/instance-cnisdxvs/file-vtnrt92f/chr20.vcf to output location +[2020-11-17T12:07:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:07:03: Ran filter-radia on TEST:chr20 successfully +[2020-11-17T12:07:03-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:07:03: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-defhtklq/cleanup/file-1fyw1v7z/stream used 123.20% (6.5 GB [7018475520B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:07:04-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-cnisdxvs +[2020-11-17T12:07:04-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-82i2tq42 with job batch system ID: 432 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:07:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8_uiyz34/worker_log.txt +[2020-11-17T12:07:05-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-82i2tq42 +[2020-11-17T12:07:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-defhtklq with job batch system ID: 433 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:07:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpcfpxy093/worker_log.txt +[2020-11-17T12:07:05-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-defhtklq +[2020-11-17T12:07:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-urgdyniq with job batch system ID: 434 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:07:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp89appwbs/worker_log.txt +[2020-11-17T12:07:06-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-urgdyniq +[2020-11-17T12:07:13-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:07:13: Ran radia on TEST:chr18 successfully +[2020-11-17T12:07:14-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-yk7gttg2 +[2020-11-17T12:07:14-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-g0mmwacm with job batch system ID: 435 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:07:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmppq1elgjy/worker_log.txt +[2020-11-17T12:07:15-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-g0mmwacm +[2020-11-17T12:07:15-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-auqpn1fd with job batch system ID: 436 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:07:15-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0nioy_pw/worker_log.txt +[2020-11-17T12:08:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:08:28: Exporting files/for-job/kind-run_filter_radia/instance-auqpn1fd/file-7rah851r/chr18.vcf to output location +[2020-11-17T12:08:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:08:28: Ran filter-radia on TEST:chr18 successfully +[2020-11-17T12:08:28-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:08:28: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-g0mmwacm/cleanup/file-ir5ie0lj/stream used 123.21% (6.5 GB [7019208704B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:08:29-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-auqpn1fd +[2020-11-17T12:08:29-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-xuilew8q with job batch system ID: 437 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:08:29-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphs9d9aoo/worker_log.txt +[2020-11-17T12:08:30-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-xuilew8q +[2020-11-17T12:08:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-g0mmwacm with job batch system ID: 438 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:08:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpqglyf7_q/worker_log.txt +[2020-11-17T12:08:30-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-g0mmwacm +[2020-11-17T12:08:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-6ja23ir9 with job batch system ID: 439 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:08:31-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpti8168pt/worker_log.txt +[2020-11-17T12:08:31-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-6ja23ir9 +[2020-11-17T12:08:39-0800] [MainThread] [I] [toil.leader] 43 jobs are running, 0 jobs are issued and waiting to run +[2020-11-17T12:08:58-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:08:58: Ran radia on TEST:chr16 successfully +[2020-11-17T12:08:59-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-8df115wr +[2020-11-17T12:08:59-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-e1lw2rb5 with job batch system ID: 440 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:08:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpotfh63s0/worker_log.txt +[2020-11-17T12:08:59-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-e1lw2rb5 +[2020-11-17T12:08:59-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-rp56hyo8 with job batch system ID: 441 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:09:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpy3g_d73x/worker_log.txt +[2020-11-17T12:09:19-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:09:19: Ran radia on TEST:chr17 successfully +[2020-11-17T12:09:20-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-l9gd4jyv +[2020-11-17T12:09:20-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-q5q4t3q4 with job batch system ID: 442 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:09:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp14wat1i6/worker_log.txt +[2020-11-17T12:09:20-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-q5q4t3q4 +[2020-11-17T12:09:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-y8pvbmow with job batch system ID: 443 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:09:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp18yn37km/worker_log.txt +[2020-11-17T12:09:51-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:09:51: Ran radia on TEST:chr15 successfully +[2020-11-17T12:09:52-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-bh6j1ee8 +[2020-11-17T12:09:52-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-_s4bq89j with job batch system ID: 444 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:09:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpux2r511r/worker_log.txt +[2020-11-17T12:09:53-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-_s4bq89j +[2020-11-17T12:09:53-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-q58uah7b with job batch system ID: 445 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:09:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3rxicwdd/worker_log.txt +[2020-11-17T12:10:19-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:10:19: Ran radia on TEST:chr13 successfully +[2020-11-17T12:10:20-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-l0fvnl_y +[2020-11-17T12:10:20-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-x9jibo3w with job batch system ID: 446 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:10:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpj91ldb5g/worker_log.txt +[2020-11-17T12:10:21-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-x9jibo3w +[2020-11-17T12:10:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-n5g7l0b5 with job batch system ID: 447 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:10:22-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:10:22: Exporting files/for-job/kind-run_filter_radia/instance-rp56hyo8/file-pascq_gr/chr16.vcf to output location +[2020-11-17T12:10:22-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:10:22: Ran filter-radia on TEST:chr16 successfully +[2020-11-17T12:10:22-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:10:22: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-e1lw2rb5/cleanup/file-t0f2cf44/stream used 123.24% (6.5 GB [7020974080B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:10:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpg3x_gagx/worker_log.txt +[2020-11-17T12:10:22-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-rp56hyo8 +[2020-11-17T12:10:22-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-5wfcyeqx with job batch system ID: 448 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:10:23-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfsrq3hre/worker_log.txt +[2020-11-17T12:10:23-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-5wfcyeqx +[2020-11-17T12:10:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-e1lw2rb5 with job batch system ID: 449 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:10:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpb35sz8xw/worker_log.txt +[2020-11-17T12:10:24-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-e1lw2rb5 +[2020-11-17T12:10:24-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-9v6yyeiq with job batch system ID: 450 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:10:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_ej0q61c/worker_log.txt +[2020-11-17T12:10:24-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-9v6yyeiq +[2020-11-17T12:10:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:10:31: Ran radia on TEST:chrX successfully +[2020-11-17T12:10:31-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-le1j906_ +[2020-11-17T12:10:31-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-atx5stn_ with job batch system ID: 451 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:10:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpg5ijwaoz/worker_log.txt +[2020-11-17T12:10:32-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-atx5stn_ +[2020-11-17T12:10:32-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-r3ge9v6h with job batch system ID: 452 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:10:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp97nrc_75/worker_log.txt +[2020-11-17T12:10:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:10:42: Exporting files/for-job/kind-run_filter_radia/instance-y8pvbmow/file-px17d9sx/chr17.vcf to output location +[2020-11-17T12:10:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:10:42: Ran filter-radia on TEST:chr17 successfully +[2020-11-17T12:10:42-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:10:42: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-q5q4t3q4/cleanup/file-wu8da09j/stream used 123.23% (6.5 GB [7020044288B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:10:43-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-y8pvbmow +[2020-11-17T12:10:43-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-3er_4t2e with job batch system ID: 453 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:10:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8ehjhwnl/worker_log.txt +[2020-11-17T12:10:44-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-3er_4t2e +[2020-11-17T12:10:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-q5q4t3q4 with job batch system ID: 454 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:10:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpay_sdwbq/worker_log.txt +[2020-11-17T12:10:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-q5q4t3q4 +[2020-11-17T12:10:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-71zlpdlm with job batch system ID: 455 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:10:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9ge_0tbd/worker_log.txt +[2020-11-17T12:10:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-71zlpdlm +[2020-11-17T12:11:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:11:15: Exporting files/for-job/kind-run_filter_radia/instance-q58uah7b/file-lcpkd6xd/chr15.vcf to output location +[2020-11-17T12:11:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:11:15: Ran filter-radia on TEST:chr15 successfully +[2020-11-17T12:11:15-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:11:15: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-_s4bq89j/cleanup/file-mk4k088w/stream used 123.19% (6.5 GB [7018209280B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:11:16-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-q58uah7b +[2020-11-17T12:11:16-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-tccq69t6 with job batch system ID: 456 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:11:17-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpbg_pxlcf/worker_log.txt +[2020-11-17T12:11:17-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-tccq69t6 +[2020-11-17T12:11:17-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-_s4bq89j with job batch system ID: 457 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:11:17-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp1bvqufes/worker_log.txt +[2020-11-17T12:11:17-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-_s4bq89j +[2020-11-17T12:11:17-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-x5vr8lls with job batch system ID: 458 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:11:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpt0ockwwi/worker_log.txt +[2020-11-17T12:11:18-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-x5vr8lls +[2020-11-17T12:11:36-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:11:36: Exporting files/for-job/kind-run_filter_radia/instance-n5g7l0b5/file-6ztt5uzk/chr13.vcf to output location +[2020-11-17T12:11:36-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:11:36: Ran filter-radia on TEST:chr13 successfully +[2020-11-17T12:11:36-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:11:36: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-x9jibo3w/cleanup/file-1itbmv1m/stream used 123.19% (6.5 GB [7017967616B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:11:37-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-n5g7l0b5 +[2020-11-17T12:11:37-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-tubbuhaf with job batch system ID: 459 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:11:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpsy_g_h93/worker_log.txt +[2020-11-17T12:11:37-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-tubbuhaf +[2020-11-17T12:11:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-x9jibo3w with job batch system ID: 460 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:11:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9bagj5c3/worker_log.txt +[2020-11-17T12:11:38-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-x9jibo3w +[2020-11-17T12:11:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-pmk6jqh0 with job batch system ID: 461 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:11:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpgj8hgjqo/worker_log.txt +[2020-11-17T12:11:39-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-pmk6jqh0 +[2020-11-17T12:11:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:11:43: Exporting files/for-job/kind-run_filter_radia/instance-r3ge9v6h/file-953tvmrn/chrX.vcf to output location +[2020-11-17T12:11:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:11:43: Ran filter-radia on TEST:chrX successfully +[2020-11-17T12:11:43-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:11:43: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-atx5stn_/cleanup/file-wbu5nras/stream used 123.20% (6.5 GB [7018274816B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:11:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-r3ge9v6h +[2020-11-17T12:11:44-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-akm1n1fr with job batch system ID: 462 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:11:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmplvb2xx6h/worker_log.txt +[2020-11-17T12:11:45-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-akm1n1fr +[2020-11-17T12:11:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-atx5stn_ with job batch system ID: 463 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:11:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvwlkqvqu/worker_log.txt +[2020-11-17T12:11:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-atx5stn_ +[2020-11-17T12:11:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-3npacuv2 with job batch system ID: 464 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:11:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpecktmtka/worker_log.txt +[2020-11-17T12:11:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-3npacuv2 +[2020-11-17T12:12:06-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:12:06: Ran radia on TEST:chr14 successfully +[2020-11-17T12:12:07-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-_7k4bhmw +[2020-11-17T12:12:07-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-voeb5_hr with job batch system ID: 465 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:12:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpt31a8ioi/worker_log.txt +[2020-11-17T12:12:08-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-voeb5_hr +[2020-11-17T12:12:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-spymmfrk with job batch system ID: 466 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:12:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4ur9xjdh/worker_log.txt +[2020-11-17T12:13:24-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:13:24: Exporting files/for-job/kind-run_filter_radia/instance-spymmfrk/file-dbtrow0b/chr14.vcf to output location +[2020-11-17T12:13:24-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:13:24: Ran filter-radia on TEST:chr14 successfully +[2020-11-17T12:13:24-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:13:24: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-voeb5_hr/cleanup/file-2yqx5eps/stream used 123.18% (6.5 GB [7017533440B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:13:25-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-spymmfrk +[2020-11-17T12:13:25-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-kal4us7g with job batch system ID: 467 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:13:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpnl6us3ts/worker_log.txt +[2020-11-17T12:13:25-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-kal4us7g +[2020-11-17T12:13:25-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-voeb5_hr with job batch system ID: 468 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:13:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfjpv2u_6/worker_log.txt +[2020-11-17T12:13:26-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-voeb5_hr +[2020-11-17T12:13:26-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-rasdqp_4 with job batch system ID: 469 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:13:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpnqjctoez/worker_log.txt +[2020-11-17T12:13:27-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-rasdqp_4 +[2020-11-17T12:14:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:14:57: Ran radia on TEST:chr9 successfully +[2020-11-17T12:14:58-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-t4meaiwm +[2020-11-17T12:14:58-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-qz9638ij with job batch system ID: 470 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:14:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp1enz_ne4/worker_log.txt +[2020-11-17T12:14:58-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-qz9638ij +[2020-11-17T12:14:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-jrcl4470 with job batch system ID: 471 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:14:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpsihzu6_6/worker_log.txt +[2020-11-17T12:16:21-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:16:21: Exporting files/for-job/kind-run_filter_radia/instance-jrcl4470/file-zd38f9h0/chr9.vcf to output location +[2020-11-17T12:16:21-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:16:21: Ran filter-radia on TEST:chr9 successfully +[2020-11-17T12:16:21-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:16:21: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-qz9638ij/cleanup/file-9u6tywka/stream used 123.19% (6.5 GB [7017967616B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:16:22-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-jrcl4470 +[2020-11-17T12:16:22-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-9bh7xk34 with job batch system ID: 472 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:16:23-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:16:23: Ran radia on TEST:chr11 successfully +[2020-11-17T12:16:23-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmppmjz6wn3/worker_log.txt +[2020-11-17T12:16:23-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-9bh7xk34 +[2020-11-17T12:16:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-qz9638ij with job batch system ID: 473 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:16:23-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-uuz6egk3 +[2020-11-17T12:16:23-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ekiyg8vs with job batch system ID: 474 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:16:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpsiqdj5no/worker_log.txt +[2020-11-17T12:16:24-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-qz9638ij +[2020-11-17T12:16:24-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-pzkcodc4 with job batch system ID: 475 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:16:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpt6mfh67t/worker_log.txt +[2020-11-17T12:16:24-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ekiyg8vs +[2020-11-17T12:16:24-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-v3mi1idw with job batch system ID: 476 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:16:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpdyn7lx2m/worker_log.txt +[2020-11-17T12:16:25-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-pzkcodc4 +[2020-11-17T12:16:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp72klm1k9/worker_log.txt +[2020-11-17T12:16:36-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:16:36: Ran radia on TEST:chr12 successfully +[2020-11-17T12:16:36-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-ih1hr4ee +[2020-11-17T12:16:36-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ghjki1n7 with job batch system ID: 477 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:16:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_l06qc_u/worker_log.txt +[2020-11-17T12:16:37-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ghjki1n7 +[2020-11-17T12:16:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-fc0ahyap with job batch system ID: 478 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:16:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpulatep2c/worker_log.txt +[2020-11-17T12:16:44-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:16:44: Ran radia on TEST:chr10 successfully +[2020-11-17T12:16:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-ozzpit9s +[2020-11-17T12:16:45-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-no2kowr1 with job batch system ID: 479 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:16:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp36tq_ubu/worker_log.txt +[2020-11-17T12:16:46-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-no2kowr1 +[2020-11-17T12:16:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-dqzpwlxg with job batch system ID: 480 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:16:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp5wsf40is/worker_log.txt +[2020-11-17T12:17:53-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:17:53: Exporting files/for-job/kind-run_filter_radia/instance-v3mi1idw/file-413nt_tk/chr11.vcf to output location +[2020-11-17T12:17:53-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:17:53: Ran filter-radia on TEST:chr11 successfully +[2020-11-17T12:17:53-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:17:53: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-ekiyg8vs/cleanup/file-a3qoa1xo/stream used 123.22% (6.5 GB [7019876352B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:17:54-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-v3mi1idw +[2020-11-17T12:17:54-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-53x21tuo with job batch system ID: 481 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:17:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfl9nb43a/worker_log.txt +[2020-11-17T12:17:54-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-53x21tuo +[2020-11-17T12:17:54-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-ekiyg8vs with job batch system ID: 482 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:17:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpub42fbe8/worker_log.txt +[2020-11-17T12:17:55-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-ekiyg8vs +[2020-11-17T12:17:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-heu34ntv with job batch system ID: 483 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:17:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpdwisroat/worker_log.txt +[2020-11-17T12:17:56-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-heu34ntv +[2020-11-17T12:18:05-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:18:05: Exporting files/for-job/kind-run_filter_radia/instance-fc0ahyap/file-_vj1nq8j/chr12.vcf to output location +[2020-11-17T12:18:05-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:18:05: Ran filter-radia on TEST:chr12 successfully +[2020-11-17T12:18:05-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:18:05: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-ghjki1n7/cleanup/file-dtgjtr11/stream used 123.21% (6.5 GB [7019073536B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:18:05-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-fc0ahyap +[2020-11-17T12:18:05-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-d80ih_n3 with job batch system ID: 484 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:18:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3hcm5qn3/worker_log.txt +[2020-11-17T12:18:06-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-d80ih_n3 +[2020-11-17T12:18:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-ghjki1n7 with job batch system ID: 485 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:18:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp1x9fladj/worker_log.txt +[2020-11-17T12:18:09-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-ghjki1n7 +[2020-11-17T12:18:09-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-2gnawt2n with job batch system ID: 486 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:18:10-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_c7df253/worker_log.txt +[2020-11-17T12:18:10-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-2gnawt2n +[2020-11-17T12:18:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:18:15: Exporting files/for-job/kind-run_filter_radia/instance-dqzpwlxg/file-3eo8m40l/chr10.vcf to output location +[2020-11-17T12:18:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:18:15: Ran filter-radia on TEST:chr10 successfully +[2020-11-17T12:18:15-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:18:15: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-no2kowr1/cleanup/file-xh4ulne3/stream used 123.22% (6.5 GB [7019663360B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:18:15-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-dqzpwlxg +[2020-11-17T12:18:15-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-bkd7a_ml with job batch system ID: 487 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:18:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpxqv95kkl/worker_log.txt +[2020-11-17T12:18:16-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-bkd7a_ml +[2020-11-17T12:18:16-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-no2kowr1 with job batch system ID: 488 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:18:17-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpuwn_pzy3/worker_log.txt +[2020-11-17T12:18:17-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-no2kowr1 +[2020-11-17T12:18:17-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-ncemqmds with job batch system ID: 489 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:18:17-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp6er5jfcn/worker_log.txt +[2020-11-17T12:18:18-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-ncemqmds +[2020-11-17T12:19:47-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:19:47: Ran radia on TEST:chr8 successfully +[2020-11-17T12:19:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-tylx98yd +[2020-11-17T12:19:48-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-8oextzp_ with job batch system ID: 490 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:19:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpaishu946/worker_log.txt +[2020-11-17T12:19:49-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-8oextzp_ +[2020-11-17T12:19:49-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-z258i7w8 with job batch system ID: 491 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:19:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpsgsi4mgt/worker_log.txt +[2020-11-17T12:21:19-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:21:19: Exporting files/for-job/kind-run_filter_radia/instance-z258i7w8/file-nmpb7jsh/chr8.vcf to output location +[2020-11-17T12:21:19-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:21:19: Ran filter-radia on TEST:chr8 successfully +[2020-11-17T12:21:19-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:21:19: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-8oextzp_/cleanup/file-n8hsnm6p/stream used 123.20% (6.5 GB [7018590208B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:21:20-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-z258i7w8 +[2020-11-17T12:21:20-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-k49n6vpb with job batch system ID: 492 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:21:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmppjeg5w_u/worker_log.txt +[2020-11-17T12:21:21-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-k49n6vpb +[2020-11-17T12:21:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-8oextzp_ with job batch system ID: 493 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:21:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp5krfmh_z/worker_log.txt +[2020-11-17T12:21:22-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-8oextzp_ +[2020-11-17T12:21:22-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-n107zzip with job batch system ID: 494 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:21:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpsj57lpxq/worker_log.txt +[2020-11-17T12:21:22-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-n107zzip +[2020-11-17T12:22:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:22:15: Ran radia on TEST:chr6 successfully +[2020-11-17T12:22:16-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-sk1hffpu +[2020-11-17T12:22:16-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-rk71u8dk with job batch system ID: 495 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:22:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpss3t14x0/worker_log.txt +[2020-11-17T12:22:17-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-rk71u8dk +[2020-11-17T12:22:17-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-bbx065uq with job batch system ID: 496 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:22:17-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp7oa0h763/worker_log.txt +[2020-11-17T12:23:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:23:45: Exporting files/for-job/kind-run_filter_radia/instance-bbx065uq/file-0tuq7g32/chr6.vcf to output location +[2020-11-17T12:23:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:23:45: Ran filter-radia on TEST:chr6 successfully +[2020-11-17T12:23:45-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:23:45: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-rk71u8dk/cleanup/file-_e_g_azy/stream used 123.23% (6.5 GB [7019929600B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:23:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-bbx065uq +[2020-11-17T12:23:45-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-aja68zx2 with job batch system ID: 497 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:23:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp2qixtbu4/worker_log.txt +[2020-11-17T12:23:46-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-aja68zx2 +[2020-11-17T12:23:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-rk71u8dk with job batch system ID: 498 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:23:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmps9b7k38p/worker_log.txt +[2020-11-17T12:23:47-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-rk71u8dk +[2020-11-17T12:23:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-hyoqamqz with job batch system ID: 499 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:23:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmptyy4cp_d/worker_log.txt +[2020-11-17T12:23:47-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-hyoqamqz +[2020-11-17T12:24:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:24:31: Ran radia on TEST:chr5 successfully +[2020-11-17T12:24:31-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-fexr4ne1 +[2020-11-17T12:24:31-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-grsl5wmz with job batch system ID: 500 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:24:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpqlvj2yw1/worker_log.txt +[2020-11-17T12:24:32-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-grsl5wmz +[2020-11-17T12:24:32-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-8la3jwf5 with job batch system ID: 501 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:24:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp93g1cmy7/worker_log.txt +[2020-11-17T12:26:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:26:01: Exporting files/for-job/kind-run_filter_radia/instance-8la3jwf5/file-v09wauwm/chr5.vcf to output location +[2020-11-17T12:26:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:26:01: Ran filter-radia on TEST:chr5 successfully +[2020-11-17T12:26:01-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:26:01: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-grsl5wmz/cleanup/file-yj6j8u3c/stream used 123.26% (6.5 GB [7021940736B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:26:01-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-8la3jwf5 +[2020-11-17T12:26:01-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-0u_k9wxz with job batch system ID: 502 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:26:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp6ca_xdru/worker_log.txt +[2020-11-17T12:26:02-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-0u_k9wxz +[2020-11-17T12:26:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-grsl5wmz with job batch system ID: 503 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:26:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmplr7v2f0q/worker_log.txt +[2020-11-17T12:26:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-grsl5wmz +[2020-11-17T12:26:03-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-vajvu1ke with job batch system ID: 504 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:26:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4d3rgwpy/worker_log.txt +[2020-11-17T12:26:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-vajvu1ke +[2020-11-17T12:26:23-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:26:23: Ran radia on TEST:chr4 successfully +[2020-11-17T12:26:23-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-y3155cnv +[2020-11-17T12:26:23-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-6q3picdy with job batch system ID: 505 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:26:24-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:26:24: Ran radia on TEST:chr7 successfully +[2020-11-17T12:26:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmppj29y7fb/worker_log.txt +[2020-11-17T12:26:24-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-6q3picdy +[2020-11-17T12:26:24-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-fdddu_a4 with job batch system ID: 506 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:26:25-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-qomyhl4z +[2020-11-17T12:26:25-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-866mv3ah with job batch system ID: 507 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:26:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9syjao6h/worker_log.txt +[2020-11-17T12:26:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpw1me773y/worker_log.txt +[2020-11-17T12:26:26-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-866mv3ah +[2020-11-17T12:26:26-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-j5lqclhk with job batch system ID: 508 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:26:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp2d5ba7by/worker_log.txt +[2020-11-17T12:27:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:27:54: Exporting files/for-job/kind-run_filter_radia/instance-fdddu_a4/file-f9nju2gc/chr4.vcf to output location +[2020-11-17T12:27:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:27:54: Ran filter-radia on TEST:chr4 successfully +[2020-11-17T12:27:54-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:27:54: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-6q3picdy/cleanup/file-g489kqqh/stream used 123.32% (6.5 GB [7025311744B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:27:55-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-fdddu_a4 +[2020-11-17T12:27:55-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-tkaozleg with job batch system ID: 509 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:27:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp7f1r7nws/worker_log.txt +[2020-11-17T12:27:56-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-tkaozleg +[2020-11-17T12:27:56-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-6q3picdy with job batch system ID: 510 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:27:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphrn_i7se/worker_log.txt +[2020-11-17T12:27:56-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-6q3picdy +[2020-11-17T12:27:56-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-kl1aviz3 with job batch system ID: 511 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:27:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpqk88cpc1/worker_log.txt +[2020-11-17T12:27:57-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-kl1aviz3 +[2020-11-17T12:28:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:28:01: Exporting files/for-job/kind-run_filter_radia/instance-j5lqclhk/file-4milbu44/chr7.vcf to output location +[2020-11-17T12:28:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:28:01: Ran filter-radia on TEST:chr7 successfully +[2020-11-17T12:28:01-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:28:01: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-866mv3ah/cleanup/file-oxugw2a5/stream used 123.20% (6.5 GB [7018774528B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:28:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-j5lqclhk +[2020-11-17T12:28:02-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-1vwigwn6 with job batch system ID: 512 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:28:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpxf_t2526/worker_log.txt +[2020-11-17T12:28:03-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-1vwigwn6 +[2020-11-17T12:28:03-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-866mv3ah with job batch system ID: 513 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:28:04-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpmwz5wd5n/worker_log.txt +[2020-11-17T12:28:04-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-866mv3ah +[2020-11-17T12:28:04-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-rut4j81p with job batch system ID: 514 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:28:04-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpv6iij83k/worker_log.txt +[2020-11-17T12:28:04-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-rut4j81p +[2020-11-17T12:29:13-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:29:13: Ran radia on TEST:chr3 successfully +[2020-11-17T12:29:13-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-31_tfqh9 +[2020-11-17T12:29:13-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-1_6ba_xv with job batch system ID: 515 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:29:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpnq5naz41/worker_log.txt +[2020-11-17T12:29:14-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-1_6ba_xv +[2020-11-17T12:29:14-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-1nxqnvdn with job batch system ID: 516 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:29:15-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpauo1s_pt/worker_log.txt +[2020-11-17T12:30:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:30:54: Exporting files/for-job/kind-run_filter_radia/instance-1nxqnvdn/file-y2wnn66o/chr3.vcf to output location +[2020-11-17T12:30:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:30:54: Ran filter-radia on TEST:chr3 successfully +[2020-11-17T12:30:54-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:30:54: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-1_6ba_xv/cleanup/file-rqycq6pz/stream used 123.23% (6.5 GB [7020032000B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:30:55-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-1nxqnvdn +[2020-11-17T12:30:55-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-12fw1st8 with job batch system ID: 517 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:30:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpiqzjy6h3/worker_log.txt +[2020-11-17T12:30:55-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-12fw1st8 +[2020-11-17T12:30:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-1_6ba_xv with job batch system ID: 518 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:30:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpia0evtmx/worker_log.txt +[2020-11-17T12:30:56-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-1_6ba_xv +[2020-11-17T12:30:56-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-3u67g6qc with job batch system ID: 519 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:30:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpn_2sgyu8/worker_log.txt +[2020-11-17T12:30:57-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-3u67g6qc +[2020-11-17T12:34:44-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:34:44: Ran radia on TEST:chr2 successfully +[2020-11-17T12:34:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-c6mc4yyt +[2020-11-17T12:34:44-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-x16svr4j with job batch system ID: 520 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:34:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp7yt_gfrd/worker_log.txt +[2020-11-17T12:34:45-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-x16svr4j +[2020-11-17T12:34:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-hh0d9gec with job batch system ID: 521 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:34:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp63_lypy9/worker_log.txt +[2020-11-17T12:36:13-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:36:13: Ran radia on TEST:chr1 successfully +[2020-11-17T12:36:14-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-1ixkm7sa +[2020-11-17T12:36:14-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-_mntndb5 with job batch system ID: 522 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:36:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3pl0b9jc/worker_log.txt +[2020-11-17T12:36:15-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-_mntndb5 +[2020-11-17T12:36:15-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-5frvl1rz with job batch system ID: 523 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-17T12:36:15-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpboz3wowi/worker_log.txt +[2020-11-17T12:36:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:36:31: Exporting files/for-job/kind-run_filter_radia/instance-hh0d9gec/file-pk_7zmpb/chr2.vcf to output location +[2020-11-17T12:36:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:36:31: Ran filter-radia on TEST:chr2 successfully +[2020-11-17T12:36:31-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:36:31: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-x16svr4j/cleanup/file-qzc5ckgg/stream used 123.24% (6.5 GB [7020888064B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:36:32-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-hh0d9gec +[2020-11-17T12:36:32-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-kwj6phg6 with job batch system ID: 524 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:36:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp44lx1ttz/worker_log.txt +[2020-11-17T12:36:33-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-kwj6phg6 +[2020-11-17T12:36:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-x16svr4j with job batch system ID: 525 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:36:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0cjygwc6/worker_log.txt +[2020-11-17T12:36:33-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-x16svr4j +[2020-11-17T12:36:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-9e1gplto with job batch system ID: 526 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:36:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp1lcm_yw9/worker_log.txt +[2020-11-17T12:36:34-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-9e1gplto +[2020-11-17T12:37:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:37:55: Exporting files/for-job/kind-run_filter_radia/instance-5frvl1rz/file-df85boaa/chr1.vcf to output location +[2020-11-17T12:37:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:37:55: Ran filter-radia on TEST:chr1 successfully +[2020-11-17T12:37:55-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-17-2020 12:37:55: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-_mntndb5/cleanup/file-pp5tt6ds/stream used 123.27% (6.5 GB [7022587904B] used, 5.3 GB [5696831657B] requested) at the end of its run. +[2020-11-17T12:37:56-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-5frvl1rz +[2020-11-17T12:37:56-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-n8tsgrru with job batch system ID: 527 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:37:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4oql7e3i/worker_log.txt +[2020-11-17T12:37:56-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-n8tsgrru +[2020-11-17T12:37:56-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-_mntndb5 with job batch system ID: 528 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:37:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpwl8vfjf9/worker_log.txt +[2020-11-17T12:37:57-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-_mntndb5 +[2020-11-17T12:37:57-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-y0a38ryy with job batch system ID: 529 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-17T12:37:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8i5fa7x1/worker_log.txt +[2020-11-17T12:37:58-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-y0a38ryy +[2020-11-17T12:37:58-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ivw_fcv_ with job batch system ID: 530 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-17T12:37:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpmku4b6x4/worker_log.txt +[2020-11-17T12:37:59-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ivw_fcv_ +[2020-11-17T13:08:41-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-17T14:08:41-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run diff --git a/errors/2020-11-17-14-13-19.txt b/errors/2020-11-17-14-13-19.txt new file mode 100644 index 00000000..dbc2b5f4 --- /dev/null +++ b/errors/2020-11-17-14-13-19.txt @@ -0,0 +1,1749 @@ +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-c60ojvxr +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ewrzdiri +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r7m8gybn +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-uuibculn +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xbtj2suv +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-9uka9hrw +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-q4vn5u6_ +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-klw60gaa +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xz7ir9sh +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-s4kmmle7 +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-v628y3w0 +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-parse_config_file/instance-rsybzib_ +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-d9qjg3jl +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-0xdl6glf +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-eirr3b8r +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-k3tx4a2p +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-y1gatlgp +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-yb2yf65u +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-etqwe7dv +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-fs46wkmz +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ge6vacd7 +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-06rjxk27 +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-mppqh5vo +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-2dtgopje +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_fusion/instance-g9_082sr +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ovr5_kcb +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-7_6wjfvk +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-eg7xdcao +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-rd57kqhd +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-jkvqxjgf +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-bv9m7hf3 +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_rsem/instance-mzzok2e3 +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-jqmhwbe8 +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-dbtlmgmn +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-exw4ljh3 +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-aux6_f19 +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-2u48n_5f +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r1uv5wsv +[2020-11-17T14:13:20-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-8p00djrs +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-klw60gaa with job batch system ID: 0 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-c60ojvxr with job batch system ID: 1 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-uuibculn with job batch system ID: 2 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-aux6_f19 with job batch system ID: 3 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-d9qjg3jl with job batch system ID: 4 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-eg7xdcao with job batch system ID: 5 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mppqh5vo with job batch system ID: 6 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yb2yf65u with job batch system ID: 7 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-q4vn5u6_ with job batch system ID: 8 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ge6vacd7 with job batch system ID: 9 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-7_6wjfvk with job batch system ID: 10 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2u48n_5f with job batch system ID: 11 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jkvqxjgf with job batch system ID: 12 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bv9m7hf3 with job batch system ID: 13 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r1uv5wsv with job batch system ID: 14 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9uka9hrw with job batch system ID: 15 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ewrzdiri with job batch system ID: 16 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ovr5_kcb with job batch system ID: 17 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xbtj2suv with job batch system ID: 18 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0xdl6glf with job batch system ID: 19 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-v628y3w0 with job batch system ID: 20 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-k3tx4a2p with job batch system ID: 21 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-06rjxk27 with job batch system ID: 22 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-rd57kqhd with job batch system ID: 23 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xz7ir9sh with job batch system ID: 24 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r7m8gybn with job batch system ID: 25 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2dtgopje with job batch system ID: 26 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-etqwe7dv with job batch system ID: 27 and cores: 80, disk: 8.2 G, and memory: 2.0 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_fusion' kind-run_fusion/instance-g9_082sr with job batch system ID: 28 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-17T14:13:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with job batch system ID: 29 and cores: 80, disk: 5.0 G, and memory: 2.0 G +[2020-11-17T14:13:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpgmwnq4c7/worker_log.txt +[2020-11-17T14:13:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp78j0ha0y/worker_log.txt +[2020-11-17T14:13:22-0800] [MainThread] [I] [toil.leader] 2 jobs are running, 28 jobs are issued and waiting to run +[2020-11-17T14:14:25-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-klw60gaa. +[2020-11-17T14:14:25-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-klw60gaa +[2020-11-17T14:14:25-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-klw60gaa +[2020-11-17T14:14:25-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-klw60gaa follows: +=========> + [2020-11-17T14:13:21-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T14:13:21-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 59, in + nthreads=int(val) + ValueError: invalid literal for int() with base 10: '80.0' + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp78j0ha0y/d1e6430c-47ab-4de3-b93f-137415c13817:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '80.0']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp78j0ha0y/d1e6430c-47ab-4de3-b93f-137415c13817:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_dna -e /home/phlat-1.0 -o /data -p 80.0" + [2020-11-17T14:14:24-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp78j0ha0y/d1e6430c-47ab-4de3-b93f-137415c13817/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp78j0ha0y/d1e6430c-47ab-4de3-b93f-137415c13817/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp78j0ha0y/d1e6430c-47ab-4de3-b93f-137415c13817/phlat_index.tar.gz'} +<========= +[2020-11-17T14:14:25-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-klw60gaa with ID kind-run_phlat/instance-klw60gaa to 1 +[2020-11-17T14:14:25-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-klw60gaa with job batch system ID: 30 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp7singr3q/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpqrn0hj2o/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpaa97kz2v/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpujdw2f7g/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8joupz3w/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpmksrz_r7/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpzohr3_qg/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpay79l4v5/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpr9m7n4ek/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_l02eo0x/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpp85vm3hi/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpu3ehzocx/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_ztrgjll/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_85bg2nt/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp85l19z7_/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9ne3rrpb/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpkow1x6mv/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_xt7qrum/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpmx_h9tr9/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp5avzsvxx/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpp_41cfiw/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpxtktkzg3/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpbpmx51ij/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpezf52tv1/worker_log.txt +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp7pf8oqkj/worker_log.txt +[2020-11-17T14:14:27-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-c60ojvxr. +[2020-11-17T14:14:27-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-c60ojvxr +[2020-11-17T14:14:27-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-c60ojvxr +[2020-11-17T14:14:27-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-c60ojvxr follows: +=========> + [2020-11-17T14:13:21-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T14:13:21-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 59, in + nthreads=int(val) + ValueError: invalid literal for int() with base 10: '80.0' + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpgmwnq4c7/d9c2892e-e5e1-4545-961c-1642dfd0f6e5:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'normal_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '80.0']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpgmwnq4c7/d9c2892e-e5e1-4545-961c-1642dfd0f6e5:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag normal_dna -e /home/phlat-1.0 -o /data -p 80.0" + [2020-11-17T14:14:25-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpgmwnq4c7/d9c2892e-e5e1-4545-961c-1642dfd0f6e5/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpgmwnq4c7/d9c2892e-e5e1-4545-961c-1642dfd0f6e5/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpgmwnq4c7/d9c2892e-e5e1-4545-961c-1642dfd0f6e5/phlat_index.tar.gz'} +<========= +[2020-11-17T14:14:27-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-c60ojvxr with ID kind-run_phlat/instance-c60ojvxr to 1 +[2020-11-17T14:14:27-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-c60ojvxr with job batch system ID: 31 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-17T14:14:28-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpn1e_ymxf/worker_log.txt +[2020-11-17T14:33:15-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_fusion file:/scratch/drkthomp/jobStore kind-run_fusion/instance-g9_082sr. +[2020-11-17T14:33:15-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-17T14:33:15-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-17T14:33:15-0800] [MainThread] [W] [toil.leader] Log from job kind-run_fusion/instance-g9_082sr follows: +=========> + [2020-11-17T14:14:28-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T14:14:28-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Error, don't understand arguments: [--CPU 80.0] at /opt/STAR-Fusion-v1.0.0/STAR-Fusion line 167. + [2020-11-17T14:33:07-0800] [MainThread] [W] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-v67rsjos/cleanup/file-d_nkrp7o/stream used 111.60% (59.6 GB [64023433216B] used, 53.4 GB [57369019908B] requested) at the end of its run. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpn1e_ymxf/3848e048-8663-4b71-9520-5a3fcdaf8942/t3zi8uf5p:/data', '--log-driver=none', 'aarjunrao/star-fusion:1.0.0', '--chimeric_junction', '/data/STAR.junction', '--output_dir', '/data/fusion-output', '--genome_lib_dir', 'star_with_fusion_100bp_readlen_index', '--CPU', '80.0']' returned non-zero exit status 255. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/fusion.py", line 118, in run_fusion + tool_version=star_fusion_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (255)for command "docker run --rm=true -v /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpn1e_ymxf/3848e048-8663-4b71-9520-5a3fcdaf8942/t3zi8uf5p:/data --log-driver=none aarjunrao/star-fusion:1.0.0 --chimeric_junction /data/STAR.junction --output_dir /data/fusion-output --genome_lib_dir star_with_fusion_100bp_readlen_index --CPU 80.0" + [2020-11-17T14:33:07-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-17T14:33:15-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_fusion' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr to 1 +[2020-11-17T14:33:15-0800] [MainThread] [I] [toil.leader] Issued job 'run_fusion' kind-run_fusion/instance-g9_082sr with job batch system ID: 32 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-17T14:33:15-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9_nts44r/worker_log.txt +[2020-11-17T14:34:25-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-etqwe7dv. +[2020-11-17T14:34:25-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-etqwe7dv +[2020-11-17T14:34:25-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-etqwe7dv +[2020-11-17T14:34:25-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-etqwe7dv follows: +=========> + [2020-11-17T14:33:15-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T14:33:15-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 59, in + nthreads=int(val) + ValueError: invalid literal for int() with base 10: '80.0' + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9_nts44r/6ded2bef-4f7e-4946-9074-222ca623f2a2:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_rna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '80.0']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9_nts44r/6ded2bef-4f7e-4946-9074-222ca623f2a2:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_rna -e /home/phlat-1.0 -o /data -p 80.0" + [2020-11-17T14:34:24-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9_nts44r/6ded2bef-4f7e-4946-9074-222ca623f2a2/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9_nts44r/6ded2bef-4f7e-4946-9074-222ca623f2a2/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9_nts44r/6ded2bef-4f7e-4946-9074-222ca623f2a2/phlat_index.tar.gz'} +<========= +[2020-11-17T14:34:25-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-etqwe7dv with ID kind-run_phlat/instance-etqwe7dv to 1 +[2020-11-17T14:34:25-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-etqwe7dv with job batch system ID: 33 and cores: 80, disk: 8.2 G, and memory: 2.0 G +[2020-11-17T14:34:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpsnevyc2e/worker_log.txt +[2020-11-17T14:35:31-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-klw60gaa. +[2020-11-17T14:35:31-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-klw60gaa +[2020-11-17T14:35:31-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-klw60gaa +[2020-11-17T14:35:31-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-klw60gaa follows: +=========> + [2020-11-17T14:34:25-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T14:34:25-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 59, in + nthreads=int(val) + ValueError: invalid literal for int() with base 10: '80.0' + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpsnevyc2e/5c6944cf-b749-4966-b133-10f770aab560:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '80.0']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpsnevyc2e/5c6944cf-b749-4966-b133-10f770aab560:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_dna -e /home/phlat-1.0 -o /data -p 80.0" + [2020-11-17T14:35:30-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpsnevyc2e/5c6944cf-b749-4966-b133-10f770aab560/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpsnevyc2e/5c6944cf-b749-4966-b133-10f770aab560/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpsnevyc2e/5c6944cf-b749-4966-b133-10f770aab560/phlat_index.tar.gz'} +<========= +[2020-11-17T14:35:31-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-klw60gaa with ID kind-run_phlat/instance-klw60gaa to 0 +[2020-11-17T14:35:31-0800] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-klw60gaa with ID kind-run_phlat/instance-klw60gaa is completely failed +[2020-11-17T14:35:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0ni3_9ps/worker_log.txt +[2020-11-17T14:36:37-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-c60ojvxr. +[2020-11-17T14:36:37-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-c60ojvxr +[2020-11-17T14:36:37-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-c60ojvxr +[2020-11-17T14:36:37-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-c60ojvxr follows: +=========> + [2020-11-17T14:35:32-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T14:35:32-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 59, in + nthreads=int(val) + ValueError: invalid literal for int() with base 10: '80.0' + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0ni3_9ps/e974fec1-2565-4085-b34b-f6b94874e942:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'normal_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '80.0']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0ni3_9ps/e974fec1-2565-4085-b34b-f6b94874e942:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag normal_dna -e /home/phlat-1.0 -o /data -p 80.0" + [2020-11-17T14:36:37-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0ni3_9ps/e974fec1-2565-4085-b34b-f6b94874e942/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0ni3_9ps/e974fec1-2565-4085-b34b-f6b94874e942/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0ni3_9ps/e974fec1-2565-4085-b34b-f6b94874e942/phlat_index.tar.gz'} +<========= +[2020-11-17T14:36:37-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-c60ojvxr with ID kind-run_phlat/instance-c60ojvxr to 0 +[2020-11-17T14:36:37-0800] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-c60ojvxr with ID kind-run_phlat/instance-c60ojvxr is completely failed +[2020-11-17T14:36:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4ydwcmvq/worker_log.txt +[2020-11-17T14:42:35-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_fusion file:/scratch/drkthomp/jobStore kind-run_fusion/instance-g9_082sr. +[2020-11-17T14:42:35-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-17T14:42:35-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-17T14:42:35-0800] [MainThread] [W] [toil.leader] Log from job kind-run_fusion/instance-g9_082sr follows: +=========> + [2020-11-17T14:36:38-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T14:36:38-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Error, don't understand arguments: [--CPU 80.0] at /opt/STAR-Fusion-v1.0.0/STAR-Fusion line 167. + [2020-11-17T14:42:28-0800] [MainThread] [W] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-v67rsjos/cleanup/file-d_nkrp7o/stream used 111.60% (59.6 GB [64023433216B] used, 53.4 GB [57369019908B] requested) at the end of its run. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4ydwcmvq/a78f3ff8-6eb9-459b-b4d6-90cadc832b60/tvh0b9kte:/data', '--log-driver=none', 'aarjunrao/star-fusion:1.0.0', '--chimeric_junction', '/data/STAR.junction', '--output_dir', '/data/fusion-output', '--genome_lib_dir', 'star_with_fusion_100bp_readlen_index', '--CPU', '80.0']' returned non-zero exit status 255. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/fusion.py", line 118, in run_fusion + tool_version=star_fusion_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (255)for command "docker run --rm=true -v /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4ydwcmvq/a78f3ff8-6eb9-459b-b4d6-90cadc832b60/tvh0b9kte:/data --log-driver=none aarjunrao/star-fusion:1.0.0 --chimeric_junction /data/STAR.junction --output_dir /data/fusion-output --genome_lib_dir star_with_fusion_100bp_readlen_index --CPU 80.0" + [2020-11-17T14:42:28-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-17T14:42:35-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_fusion' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr to 0 +[2020-11-17T14:42:35-0800] [MainThread] [W] [toil.leader] Job 'run_fusion' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr is completely failed +[2020-11-17T14:42:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpnht95ei3/worker_log.txt +[2020-11-17T14:43:44-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-etqwe7dv. +[2020-11-17T14:43:44-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-etqwe7dv +[2020-11-17T14:43:44-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-etqwe7dv +[2020-11-17T14:43:44-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-etqwe7dv follows: +=========> + [2020-11-17T14:42:35-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T14:42:35-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 59, in + nthreads=int(val) + ValueError: invalid literal for int() with base 10: '80.0' + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpnht95ei3/0866ae0a-7eb0-4c7d-b969-34b5ca857827:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_rna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '80.0']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpnht95ei3/0866ae0a-7eb0-4c7d-b969-34b5ca857827:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_rna -e /home/phlat-1.0 -o /data -p 80.0" + [2020-11-17T14:43:43-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpnht95ei3/0866ae0a-7eb0-4c7d-b969-34b5ca857827/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpnht95ei3/0866ae0a-7eb0-4c7d-b969-34b5ca857827/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpnht95ei3/0866ae0a-7eb0-4c7d-b969-34b5ca857827/phlat_index.tar.gz'} +<========= +[2020-11-17T14:43:44-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-etqwe7dv with ID kind-run_phlat/instance-etqwe7dv to 0 +[2020-11-17T14:43:44-0800] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-etqwe7dv with ID kind-run_phlat/instance-etqwe7dv is completely failed +[2020-11-17T14:43:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpklq9f5bt/worker_log.txt +[2020-11-17T14:44:37-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_rsem file:/scratch/drkthomp/jobStore kind-run_rsem/instance-mzzok2e3. +[2020-11-17T14:44:37-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-17T14:44:37-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-17T14:44:37-0800] [MainThread] [W] [toil.leader] Log from job kind-run_rsem/instance-mzzok2e3 follows: +=========> + [2020-11-17T14:43:44-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T14:43:44-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Value "80.0" invalid for option p (number expected) + NAME + rsem-calculate-expression + + SYNOPSIS + rsem-calculate-expression [options] upstream_read_file(s) reference_name sample_name + rsem-calculate-expression [options] --paired-end upstream_read_file(s) downstream_read_file(s) reference_name sample_name + rsem-calculate-expression [options] --sam/--bam [--paired-end] input reference_name sample_name + + ARGUMENTS + upstream_read_files(s) + Comma-separated list of files containing single-end reads or + upstream reads for paired-end data. By default, these files are + assumed to be in FASTQ format. If the --no-qualities option is + specified, then FASTA format is expected. + + downstream_read_file(s) + Comma-separated list of files containing downstream reads which are + paired with the upstream reads. By default, these files are assumed + to be in FASTQ format. If the --no-qualities option is specified, + then FASTA format is expected. + + input + SAM/BAM formatted input file. If "-" is specified for the filename, + SAM/BAM input is instead assumed to come from standard input. RSEM + requires all alignments of the same read group together. For + paired-end reads, RSEM also requires the two mates of any alignment + be adjacent. See Description section for how to make input file obey + RSEM's requirements. + + reference_name + The name of the reference used. The user must have run + 'rsem-prepare-reference' with this reference_name before running + this program. + + sample_name + The name of the sample analyzed. All output files are prefixed by + this name (e.g., sample_name.genes.results) + + BASIC OPTIONS + --paired-end + Input reads are paired-end reads. (Default: off) + + --no-qualities + Input reads do not contain quality scores. (Default: off) + + --strand-specific + The RNA-Seq protocol used to generate the reads is strand specific, + i.e., all (upstream) reads are derived from the forward strand. This + option is equivalent to --forward-prob=1.0. With this option set, if + RSEM runs the Bowtie/Bowtie 2 aligner, the '--norc' Bowtie/Bowtie 2 + option will be used, which disables alignment to the reverse strand + of transcripts. (Default: off) + + --bowtie2 + Use Bowtie 2 instead of Bowtie to align reads. Since currently RSEM + does not handle indel, local and discordant alignments, the Bowtie2 + parameters are set in a way to avoid those alignments. In + particular, we use options '--sensitive --dpad 0 --gbar 99999999 + --mp 1,1 --np 1 --score-min L,0,-0.1' by default. The last parameter + of '--score-min', '-0.1', is the negative of maximum mismatch rate. + This rate can be set by option '--bowtie2-mismatch-rate'. If reads + are paired-end, we additionally use options '--no-mixed' and + '--no-discordant'. (Default: off) + + --sam + Input file is in SAM format. (Default: off) + + --bam + Input file is in BAM format. (Default: off) + + -p/--num-threads + Number of threads to use. Both Bowtie/Bowtie2, expression estimation + and 'samtools sort' will use this many threads. (Default: 1) + + --no-bam-output + Do not output any BAM file. (Default: off) + + --output-genome-bam + Generate a BAM file, 'sample_name.genome.bam', with alignments + mapped to genomic coordinates and annotated with their posterior + probabilities. In addition, RSEM will call samtools (included in + RSEM package) to sort and index the bam file. + 'sample_name.genome.sorted.bam' and + 'sample_name.genome.sorted.bam.bai' will be generated. (Default: + off) + + --sampling-for-bam + When RSEM generates a BAM file, instead of outputing all alignments + a read has with their posterior probabilities, one alignment is + sampled according to the posterior probabilities. The sampling + procedure includes the alignment to the "noise" transcript, which + does not appear in the BAM file. Only the sampled alignment has a + weight of 1. All other alignments have weight 0. If the "noise" + transcript is sampled, all alignments appeared in the BAM file + should have weight 0. (Default: off) + + --seed + Set the seed for the random number generators used in calculating + posterior mean estimates and credibility intervals. The seed must be + a non-negative 32 bit interger. (Default: off) + + --calc-pme + Run RSEM's collapsed Gibbs sampler to calculate posterior mean + estimates. (Default: off) + + --calc-ci + Calculate 95% credibility intervals and posterior mean estimates. + The credibility level can be changed by setting + '--ci-credibility-level'. (Default: off) + + -q/--quiet + Suppress the output of logging information. (Default: off) + + -h/--help + Show help information. + + --version + Show version information. + + ADVANCED OPTIONS + --sam-header-info + RSEM reads header information from input by default. If this option + is on, header information is read from the specified file. For the + format of the file, please see SAM official website. (Default: "") + + --seed-length + Seed length used by the read aligner. Providing the correct value is + important for RSEM. If RSEM runs Bowtie, it uses this value for + Bowtie's seed length parameter. Any read with its or at least one of + its mates' (for paired-end reads) length less than this value will + be ignored. If the references are not added poly(A) tails, the + minimum allowed value is 5, otherwise, the minimum allowed value is + 25. Note that this script will only check if the value >= 5 and give + a warning message if the value < 25 but >= 5. (Default: 25) + + --tag + The name of the optional field used in the SAM input for identifying + a read with too many valid alignments. The field should have the + format :i:, where a bigger than 0 indicates + a read with too many alignments. (Default: "") + + --bowtie-path + The path to the Bowtie executables. (Default: the path to the Bowtie + executables is assumed to be in the user's PATH environment + variable) + + --bowtie-n + (Bowtie parameter) max # of mismatches in the seed. (Range: 0-3, + Default: 2) + + --bowtie-e + (Bowtie parameter) max sum of mismatch quality scores across the + alignment. (Default: 99999999) + + --bowtie-m + (Bowtie parameter) suppress all alignments for a read if > + valid alignments exist. (Default: 200) + + --bowtie-chunkmbs + (Bowtie parameter) memory allocated for best first alignment + calculation (Default: 0 - use Bowtie's default) + + --phred33-quals + Input quality scores are encoded as Phred+33. (Default: on) + + --phred64-quals + Input quality scores are encoded as Phred+64 (default for GA + Pipeline ver. >= 1.3). (Default: off) + + --solexa-quals + Input quality scores are solexa encoded (from GA Pipeline ver. < + 1.3). (Default: off) + + --bowtie2-path + (Bowtie 2 parameter) The path to the Bowtie 2 executables. (Default: + the path to the Bowtie 2 executables is assumed to be in the user's + PATH environment variable) + + --bowtie2-mismatch-rate + (Bowtie 2 parameter) The maximum mismatch rate allowed. (Default: + 0.1) + + --bowtie2-k + (Bowtie 2 parameter) Find up to alignments per read. (Default: + 200) + + --bowtie2-sensitivity-level + (Bowtie 2 parameter) Set Bowtie 2's preset options in --end-to-end + mode. This option controls how hard Bowtie 2 tries to find + alignments. must be one of "very_fast", "fast", "sensitive" + and "very_sensitive". The four candidates correspond to Bowtie 2's + "--very-fast", "--fast", "--sensitive" and "--very-sensitive" + options. (Default: "sensitive" - use Bowtie 2's default) + + --forward-prob + Probability of generating a read from the forward strand of a + transcript. Set to 1 for a strand-specific protocol where all + (upstream) reads are derived from the forward strand, 0 for a + strand-specific protocol where all (upstream) read are derived from + the reverse strand, or 0.5 for a non-strand-specific protocol. + (Default: 0.5) + + --fragment-length-min + Minimum read/insert length allowed. This is also the value for the + Bowtie/Bowtie2 -I option. (Default: 1) + + --fragment-length-max + Maximum read/insert length allowed. This is also the value for the + Bowtie/Bowtie 2 -X option. (Default: 1000) + + --fragment-length-mean + (single-end data only) The mean of the fragment length distribution, + which is assumed to be a Gaussian. (Default: -1, which disables use + of the fragment length distribution) + + --fragment-length-sd + (single-end data only) The standard deviation of the fragment length + distribution, which is assumed to be a Gaussian. (Default: 0, which + assumes that all fragments are of the same length, given by the + rounded value of --fragment-length-mean) + + --estimate-rspd + Set this option if you want to estimate the read start position + distribution (RSPD) from data. Otherwise, RSEM will use a uniform + RSPD. (Default: off) + + --num-rspd-bins + Number of bins in the RSPD. Only relevant when '--estimate-rspd' is + specified. Use of the default setting is recommended. (Default: 20) + + --gibbs-burnin + The number of burn-in rounds for RSEM's Gibbs sampler. Each round + passes over the entire data set once. If RSEM can use multiple + threads, multiple Gibbs samplers will start at the same time and all + samplers share the same burn-in number. (Default: 200) + + --gibbs-number-of-samples + The total number of count vectors RSEM will collect from its Gibbs + samplers. (Default: 1000) + + --gibbs-sampling-gap + The number of rounds between two succinct count vectors RSEM + collects. If the count vector after round N is collected, the count + vector after round N + will also be collected. (Default: 1) + + --ci-credibility-level + The credibility level for credibility intervals. (Default: 0.95) + + --ci-memory + Maximum size (in memory, MB) of the auxiliary buffer used for + computing credibility intervals (CI). Set it larger for a faster CI + calculation. However, leaving 2 GB memory free for other usage is + recommended. (Default: 1024) + + --ci-number-of-samples-per-count-vector + The number of read generating probability vectors sampled per + sampled count vector. The crebility intervals are calculated by + first sampling P(C | D) and then sampling P(Theta | C) for each + sampled count vector. This option controls how many Theta vectors + are sampled per sampled count vector. (Default: 50) + + --samtools-sort-mem + Set the maximum memory per thread that can be used by 'samtools + sort'. represents the memory and accepts suffices 'K/M/G'. + RSEM will pass to the '-m' option of 'samtools sort'. + Please note that the default used here is different from the default + used by samtools. (Default: 1G) + + --keep-intermediate-files + Keep temporary files generated by RSEM. RSEM creates a temporary + directory, 'sample_name.temp', into which it puts all intermediate + output files. If this directory already exists, RSEM overwrites all + files generated by previous RSEM runs inside of it. By default, + after RSEM finishes, the temporary directory is deleted. Set this + option to prevent the deletion of this directory and the + intermediate files inside of it. (Default: off) + + --temporary-folder + Set where to put the temporary files generated by RSEM. If the + folder specified does not exist, RSEM will try to create it. + (Default: sample_name.temp) + + --time + Output time consumed by each step of RSEM to 'sample_name.time'. + (Default: off) + + DESCRIPTION + In its default mode, this program aligns input reads against a reference + transcriptome with Bowtie and calculates expression values using the + alignments. RSEM assumes the data are single-end reads with quality + scores, unless the '--paired-end' or '--no-qualities' options are + specified. Users may use an alternative aligner by specifying one of the + --sam and --bam options, and providing an alignment file in the + specified format. However, users should make sure that they align + against the indices generated by 'rsem-prepare-reference' and the + alignment file satisfies the requirements mentioned in ARGUMENTS + section. + + One simple way to make the alignment file satisfying RSEM's requirements + (assuming the aligner used put mates in a paired-end read adjacent) is + to use 'convert-sam-for-rsem' script. This script only accept SAM format + files as input. If a BAM format file is obtained, please use samtools to + convert it to a SAM file first. For example, if '/ref/mouse_125' is the + 'reference_name' and the SAM file is named 'input.sam', you can run the + following command: + + convert-sam-for-rsem /ref/mouse_125 input.sam -o input_for_rsem.sam + + For details, please refer to 'convert-sam-for-rsem's documentation page. + + The SAM/BAM format RSEM uses is v1.4. However, it is compatible with old + SAM/BAM format. However, RSEM cannot recognize 0x100 in the FLAG field. + In addition, RSEM requires SEQ and QUAL are not '*'. + + The user must run 'rsem-prepare-reference' with the appropriate + reference before using this program. + + For single-end data, it is strongly recommended that the user provide + the fragment length distribution parameters (--fragment-length-mean and + --fragment-length-sd). For paired-end data, RSEM will automatically + learn a fragment length distribution from the data. + + Please note that some of the default values for the Bowtie parameters + are not the same as those defined for Bowtie itself. + + The temporary directory and all intermediate files will be removed when + RSEM finishes unless '--keep-intermediate-files' is specified. + + With the '--calc-pme' option, posterior mean estimates will be + calculated in addition to maximum likelihood estimates. + + With the '--calc-ci' option, 95% credibility intervals and posterior + mean estimates will be calculated in addition to maximum likelihood + estimates. + + OUTPUT + sample_name.isoforms.results + File containing isoform level expression estimates. The first line + contains column names separated by the tab character. The format of + each line in the rest of this file is: + + transcript_id gene_id length effective_length expected_count TPM + FPKM IsoPct [posterior_mean_count + posterior_standard_deviation_of_count pme_TPM pme_FPKM + IsoPct_from_pme_TPM TPM_ci_lower_bound TPM_ci_upper_bound + FPKM_ci_lower_bound FPKM_ci_upper_bound] + + Fields are separated by the tab character. Fields within "[]" are + optional. They will not be presented if neither '--calc-pme' nor + '--calc-ci' is set. + + 'transcript_id' is the transcript name of this transcript. 'gene_id' + is the gene name of the gene which this transcript belongs to + (denote this gene as its parent gene). If no gene information is + provided, 'gene_id' and 'transcript_id' are the same. + + 'length' is this transcript's sequence length (poly(A) tail is not + counted). 'effective_length' counts only the positions that can + generate a valid fragment. If no poly(A) tail is added, + 'effective_length' is equal to transcript length - mean fragment + length + 1. If one transcript's effective length is less than 1, + this transcript's both effective length and abundance estimates are + set to 0. + + 'expected_count' is the sum of the posterior probability of each + read comes from this transcript over all reads. Because 1) each read + aligning to this transcript has a probability of being generated + from background noise; 2) RSEM may filter some alignable low quality + reads, the sum of expected counts for all transcript are generally + less than the total number of reads aligned. + + 'TPM' stands for Transcripts Per Million. It is a relative measure + of transcript abundance. The sum of all transcripts' TPM is 1 + million. 'FPKM' stands for Fragments Per Kilobase of transcript per + Million mapped reads. It is another relative measure of transcript + abundance. If we define l_bar be the mean transcript length in a + sample, which can be calculated as + + l_bar = \sum_i TPM_i / 10^6 * effective_length_i (i goes through + every transcript), + + the following equation is hold: + + FPKM_i = 10^3 / l_bar * TPM_i. + + We can see that the sum of FPKM is not a constant across samples. + + 'IsoPct' stands for isoform percentage. It is the percentage of this + transcript's abandunce over its parent gene's abandunce. If its + parent gene has only one isoform or the gene information is not + provided, this field will be set to 100. + + 'posterior_mean_count', 'pme_TPM', 'pme_FPKM' are posterior mean + estimates calculated by RSEM's Gibbs sampler. + 'posterior_standard_deviation_of_count' is the posterior standard + deviation of counts. 'IsoPct_from_pme_TPM' is the isoform percentage + calculated from 'pme_TPM' values. + + 'TPM_ci_lower_bound', 'TPM_ci_upper_bound', 'FPKM_ci_lower_bound' + and 'FPKM_ci_upper_bound' are lower(l) and upper(u) bounds of 95% + credibility intervals for TPM and FPKM values. The bounds are + inclusive (i.e. [l, u]). + + sample_name.genes.results + File containing gene level expression estimates. The first line + contains column names separated by the tab character. The format of + each line in the rest of this file is: + + gene_id transcript_id(s) length effective_length expected_count TPM + FPKM [posterior_mean_count posterior_standard_deviation_of_count + pme_TPM pme_FPKM TPM_ci_lower_bound TPM_ci_upper_bound + FPKM_ci_lower_bound FPKM_ci_upper_bound] + + Fields are separated by the tab character. Fields within "[]" are + optional. They will not be presented if neither '--calc-pme' nor + '--calc-ci' is set. + + 'transcript_id(s)' is a comma-separated list of transcript_ids + belonging to this gene. If no gene information is provided, + 'gene_id' and 'transcript_id(s)' are identical (the + 'transcript_id'). + + A gene's 'length' and 'effective_length' are defined as the weighted + average of its transcripts' lengths and effective lengths (weighted + by 'IsoPct'). A gene's abundance estimates are just the sum of its + transcripts' abundance estimates. + + sample_name.alleles.results + Only generated when the RSEM references are built with + allele-specific transcripts. + + This file contains allele level expression estimates for + allele-specific expression calculation. The first line contains + column names separated by the tab character. The format of each line + in the rest of this file is: + + allele_id transcript_id gene_id length effective_length + expected_count TPM FPKM AlleleIsoPct AlleleGenePct + [posterior_mean_count posterior_standard_deviation_of_count pme_TPM + pme_FPKM AlleleIsoPct_from_pme_TPM AlleleGenePct_from_pme_TPM + TPM_ci_lower_bound TPM_ci_upper_bound FPKM_ci_lower_bound + FPKM_ci_upper_bound] + + Fields are separated by the tab character. Fields within "[]" are + optional. They will not be presented if neither '--calc-pme' nor + '--calc-ci' is set. + + 'allele_id' is the allele-specific name of this allele-specific + transcript. + + 'AlleleIsoPct' stands for allele-specific percentage on isoform + level. It is the percentage of this allele-specific transcript's + abundance over its parent transcript's abundance. If its parent + transcript has only one allele variant form, this field will be set + to 100. + + 'AlleleGenePct' stands for allele-specific percentage on gene level. + It is the percentage of this allele-specific transcript's abundance + over its parent gene's abundance. + + 'AlleleIsoPct_from_pme_TPM' and 'AlleleGenePct_from_pme_TPM' have + similar meanings. They are calculated based on posterior mean + estimates. + + Please note that if this file is present, the fields 'length' and + 'effective_length' in 'sample_name.isoforms.results' should be + interpreted similarly as the corresponding definitions in + 'sample_name.genes.results'. + + sample_name.transcript.bam, sample_name.transcript.sorted.bam and + sample_name.transcript.sorted.bam.bai + Only generated when --no-bam-output is not specified. + + 'sample_name.transcript.bam' is a BAM-formatted file of read + alignments in transcript coordinates. The MAPQ field of each + alignment is set to min(100, floor(-10 * log10(1.0 - w) + 0.5)), + where w is the posterior probability of that alignment being the + true mapping of a read. In addition, RSEM pads a new tag ZW:f:value, + where value is a single precision floating number representing the + posterior probability. Because this file contains all alignment + lines produced by bowtie or user-specified aligners, it can also be + used as a replacement of the aligner generated BAM/SAM file. For + paired-end reads, if one mate has alignments but the other does not, + this file marks the alignable mate as "unmappable" (flag bit 0x4) + and appends an optional field "Z0:A:!". + + 'sample_name.transcript.sorted.bam' and + 'sample_name.transcript.sorted.bam.bai' are the sorted BAM file and + indices generated by samtools (included in RSEM package). + + sample_name.genome.bam, sample_name.genome.sorted.bam and + sample_name.genome.sorted.bam.bai + Only generated when --no-bam-output is not specified and + --output-genome-bam is specified. + + 'sample_name.genome.bam' is a BAM-formatted file of read alignments + in genomic coordinates. Alignments of reads that have identical + genomic coordinates (i.e., alignments to different isoforms that + share the same genomic region) are collapsed into one alignment. The + MAPQ field of each alignment is set to min(100, floor(-10 * + log10(1.0 - w) + 0.5)), where w is the posterior probability of that + alignment being the true mapping of a read. In addition, RSEM pads a + new tag ZW:f:value, where value is a single precision floating + number representing the posterior probability. If an alignment is + spliced, a XS:A:value tag is also added, where value is either '+' + or '-' indicating the strand of the transcript it aligns to. + + 'sample_name.genome.sorted.bam' and + 'sample_name.genome.sorted.bam.bai' are the sorted BAM file and + indices generated by samtools (included in RSEM package). + + sample_name.time + Only generated when --time is specified. + + It contains time (in seconds) consumed by aligning reads, estimating + expression levels and calculating credibility intervals. + + sample_name.stat + This is a folder instead of a file. All model related statistics are + stored in this folder. Use 'rsem-plot-model' can generate plots + using this folder. + + 'sample_name.stat/sample_name.cnt' contains alignment statistics. + The format and meanings of each field are described in + 'cnt_file_description.txt' under RSEM directory. + + 'sample_name.stat/sample_name.model' stores RNA-Seq model parameters + learned from the data. The format and meanings of each filed of this + file are described in 'model_file_description.txt' under RSEM + directory. + + EXAMPLES + Assume the path to the bowtie executables is in the user's PATH + environment variable. Reference files are under '/ref' with name + 'mouse_125'. + + 1) '/data/mmliver.fq', single-end reads with quality scores. Quality + scores are encoded as for 'GA pipeline version >= 1.3'. We want to use 8 + threads and generate a genome BAM file: + + rsem-calculate-expression --phred64-quals \ + -p 8 \ + --output-genome-bam \ + /data/mmliver.fq \ + /ref/mouse_125 \ + mmliver_single_quals + + 2) '/data/mmliver_1.fq' and '/data/mmliver_2.fq', paired-end reads with + quality scores. Quality scores are in SANGER format. We want to use 8 + threads and do not generate a genome BAM file: + + rsem-calculate-expression -p 8 \ + --paired-end \ + /data/mmliver_1.fq \ + /data/mmliver_2.fq \ + /ref/mouse_125 \ + mmliver_paired_end_quals + + 3) '/data/mmliver.fa', single-end reads without quality scores. We want + to use 8 threads: + + rsem-calculate-expression -p 8 \ + --no-qualities \ + /data/mmliver.fa \ + /ref/mouse_125 \ + mmliver_single_without_quals + + 4) Data are the same as 1). This time we assume the bowtie executables + are under '/sw/bowtie'. We want to take a fragment length distribution + into consideration. We set the fragment length mean to 150 and the + standard deviation to 35. In addition to a BAM file, we also want to + generate credibility intervals. We allow RSEM to use 1GB of memory for + CI calculation: + + rsem-calculate-expression --bowtie-path /sw/bowtie \ + --phred64-quals \ + --fragment-length-mean 150.0 \ + --fragment-length-sd 35.0 \ + -p 8 \ + --output-genome-bam \ + --calc-ci \ + --ci-memory 1024 \ + /data/mmliver.fq \ + /ref/mouse_125 \ + mmliver_single_quals + + 5) '/data/mmliver_paired_end_quals.bam', paired-end reads with quality + scores. We want to use 8 threads: + + rsem-calculate-expression --paired-end \ + --bam \ + -p 8 \ + /data/mmliver_paired_end_quals.bam \ + /ref/mouse_125 \ + mmliver_paired_end_quals + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpklq9f5bt/c8abb2e4-e63e-41e3-86a9-27160954a03a:/data', '--log-driver=none', 'aarjunrao/rsem:1.2.20', '--paired-end', '-p', '80.0', '--bam', '/data/star_transcriptome.bam', '--no-bam-output', '/data/rsem_indexes/hg38', 'rsem']' returned non-zero exit status 2. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/expression_profiling/rsem.py", line 88, in run_rsem + dockerhub=univ_options['dockerhub'], tool_version=rsem_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (2)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpklq9f5bt/c8abb2e4-e63e-41e3-86a9-27160954a03a:/data --log-driver=none aarjunrao/rsem:1.2.20 --paired-end -p 80.0 --bam /data/star_transcriptome.bam --no-bam-output /data/rsem_indexes/hg38 rsem" + [2020-11-17T14:44:36-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-17T14:44:37-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with ID kind-run_rsem/instance-mzzok2e3 to 1 +[2020-11-17T14:44:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with job batch system ID: 34 and cores: 80, disk: 5.0 G, and memory: 2.0 G +[2020-11-17T14:44:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpcqum6ex8/worker_log.txt +[2020-11-17T14:45:34-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_rsem file:/scratch/drkthomp/jobStore kind-run_rsem/instance-mzzok2e3. +[2020-11-17T14:45:34-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-17T14:45:34-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-17T14:45:34-0800] [MainThread] [W] [toil.leader] Log from job kind-run_rsem/instance-mzzok2e3 follows: +=========> + [2020-11-17T14:44:38-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-17T14:44:38-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Value "80.0" invalid for option p (number expected) + NAME + rsem-calculate-expression + + SYNOPSIS + rsem-calculate-expression [options] upstream_read_file(s) reference_name sample_name + rsem-calculate-expression [options] --paired-end upstream_read_file(s) downstream_read_file(s) reference_name sample_name + rsem-calculate-expression [options] --sam/--bam [--paired-end] input reference_name sample_name + + ARGUMENTS + upstream_read_files(s) + Comma-separated list of files containing single-end reads or + upstream reads for paired-end data. By default, these files are + assumed to be in FASTQ format. If the --no-qualities option is + specified, then FASTA format is expected. + + downstream_read_file(s) + Comma-separated list of files containing downstream reads which are + paired with the upstream reads. By default, these files are assumed + to be in FASTQ format. If the --no-qualities option is specified, + then FASTA format is expected. + + input + SAM/BAM formatted input file. If "-" is specified for the filename, + SAM/BAM input is instead assumed to come from standard input. RSEM + requires all alignments of the same read group together. For + paired-end reads, RSEM also requires the two mates of any alignment + be adjacent. See Description section for how to make input file obey + RSEM's requirements. + + reference_name + The name of the reference used. The user must have run + 'rsem-prepare-reference' with this reference_name before running + this program. + + sample_name + The name of the sample analyzed. All output files are prefixed by + this name (e.g., sample_name.genes.results) + + BASIC OPTIONS + --paired-end + Input reads are paired-end reads. (Default: off) + + --no-qualities + Input reads do not contain quality scores. (Default: off) + + --strand-specific + The RNA-Seq protocol used to generate the reads is strand specific, + i.e., all (upstream) reads are derived from the forward strand. This + option is equivalent to --forward-prob=1.0. With this option set, if + RSEM runs the Bowtie/Bowtie 2 aligner, the '--norc' Bowtie/Bowtie 2 + option will be used, which disables alignment to the reverse strand + of transcripts. (Default: off) + + --bowtie2 + Use Bowtie 2 instead of Bowtie to align reads. Since currently RSEM + does not handle indel, local and discordant alignments, the Bowtie2 + parameters are set in a way to avoid those alignments. In + particular, we use options '--sensitive --dpad 0 --gbar 99999999 + --mp 1,1 --np 1 --score-min L,0,-0.1' by default. The last parameter + of '--score-min', '-0.1', is the negative of maximum mismatch rate. + This rate can be set by option '--bowtie2-mismatch-rate'. If reads + are paired-end, we additionally use options '--no-mixed' and + '--no-discordant'. (Default: off) + + --sam + Input file is in SAM format. (Default: off) + + --bam + Input file is in BAM format. (Default: off) + + -p/--num-threads + Number of threads to use. Both Bowtie/Bowtie2, expression estimation + and 'samtools sort' will use this many threads. (Default: 1) + + --no-bam-output + Do not output any BAM file. (Default: off) + + --output-genome-bam + Generate a BAM file, 'sample_name.genome.bam', with alignments + mapped to genomic coordinates and annotated with their posterior + probabilities. In addition, RSEM will call samtools (included in + RSEM package) to sort and index the bam file. + 'sample_name.genome.sorted.bam' and + 'sample_name.genome.sorted.bam.bai' will be generated. (Default: + off) + + --sampling-for-bam + When RSEM generates a BAM file, instead of outputing all alignments + a read has with their posterior probabilities, one alignment is + sampled according to the posterior probabilities. The sampling + procedure includes the alignment to the "noise" transcript, which + does not appear in the BAM file. Only the sampled alignment has a + weight of 1. All other alignments have weight 0. If the "noise" + transcript is sampled, all alignments appeared in the BAM file + should have weight 0. (Default: off) + + --seed + Set the seed for the random number generators used in calculating + posterior mean estimates and credibility intervals. The seed must be + a non-negative 32 bit interger. (Default: off) + + --calc-pme + Run RSEM's collapsed Gibbs sampler to calculate posterior mean + estimates. (Default: off) + + --calc-ci + Calculate 95% credibility intervals and posterior mean estimates. + The credibility level can be changed by setting + '--ci-credibility-level'. (Default: off) + + -q/--quiet + Suppress the output of logging information. (Default: off) + + -h/--help + Show help information. + + --version + Show version information. + + ADVANCED OPTIONS + --sam-header-info + RSEM reads header information from input by default. If this option + is on, header information is read from the specified file. For the + format of the file, please see SAM official website. (Default: "") + + --seed-length + Seed length used by the read aligner. Providing the correct value is + important for RSEM. If RSEM runs Bowtie, it uses this value for + Bowtie's seed length parameter. Any read with its or at least one of + its mates' (for paired-end reads) length less than this value will + be ignored. If the references are not added poly(A) tails, the + minimum allowed value is 5, otherwise, the minimum allowed value is + 25. Note that this script will only check if the value >= 5 and give + a warning message if the value < 25 but >= 5. (Default: 25) + + --tag + The name of the optional field used in the SAM input for identifying + a read with too many valid alignments. The field should have the + format :i:, where a bigger than 0 indicates + a read with too many alignments. (Default: "") + + --bowtie-path + The path to the Bowtie executables. (Default: the path to the Bowtie + executables is assumed to be in the user's PATH environment + variable) + + --bowtie-n + (Bowtie parameter) max # of mismatches in the seed. (Range: 0-3, + Default: 2) + + --bowtie-e + (Bowtie parameter) max sum of mismatch quality scores across the + alignment. (Default: 99999999) + + --bowtie-m + (Bowtie parameter) suppress all alignments for a read if > + valid alignments exist. (Default: 200) + + --bowtie-chunkmbs + (Bowtie parameter) memory allocated for best first alignment + calculation (Default: 0 - use Bowtie's default) + + --phred33-quals + Input quality scores are encoded as Phred+33. (Default: on) + + --phred64-quals + Input quality scores are encoded as Phred+64 (default for GA + Pipeline ver. >= 1.3). (Default: off) + + --solexa-quals + Input quality scores are solexa encoded (from GA Pipeline ver. < + 1.3). (Default: off) + + --bowtie2-path + (Bowtie 2 parameter) The path to the Bowtie 2 executables. (Default: + the path to the Bowtie 2 executables is assumed to be in the user's + PATH environment variable) + + --bowtie2-mismatch-rate + (Bowtie 2 parameter) The maximum mismatch rate allowed. (Default: + 0.1) + + --bowtie2-k + (Bowtie 2 parameter) Find up to alignments per read. (Default: + 200) + + --bowtie2-sensitivity-level + (Bowtie 2 parameter) Set Bowtie 2's preset options in --end-to-end + mode. This option controls how hard Bowtie 2 tries to find + alignments. must be one of "very_fast", "fast", "sensitive" + and "very_sensitive". The four candidates correspond to Bowtie 2's + "--very-fast", "--fast", "--sensitive" and "--very-sensitive" + options. (Default: "sensitive" - use Bowtie 2's default) + + --forward-prob + Probability of generating a read from the forward strand of a + transcript. Set to 1 for a strand-specific protocol where all + (upstream) reads are derived from the forward strand, 0 for a + strand-specific protocol where all (upstream) read are derived from + the reverse strand, or 0.5 for a non-strand-specific protocol. + (Default: 0.5) + + --fragment-length-min + Minimum read/insert length allowed. This is also the value for the + Bowtie/Bowtie2 -I option. (Default: 1) + + --fragment-length-max + Maximum read/insert length allowed. This is also the value for the + Bowtie/Bowtie 2 -X option. (Default: 1000) + + --fragment-length-mean + (single-end data only) The mean of the fragment length distribution, + which is assumed to be a Gaussian. (Default: -1, which disables use + of the fragment length distribution) + + --fragment-length-sd + (single-end data only) The standard deviation of the fragment length + distribution, which is assumed to be a Gaussian. (Default: 0, which + assumes that all fragments are of the same length, given by the + rounded value of --fragment-length-mean) + + --estimate-rspd + Set this option if you want to estimate the read start position + distribution (RSPD) from data. Otherwise, RSEM will use a uniform + RSPD. (Default: off) + + --num-rspd-bins + Number of bins in the RSPD. Only relevant when '--estimate-rspd' is + specified. Use of the default setting is recommended. (Default: 20) + + --gibbs-burnin + The number of burn-in rounds for RSEM's Gibbs sampler. Each round + passes over the entire data set once. If RSEM can use multiple + threads, multiple Gibbs samplers will start at the same time and all + samplers share the same burn-in number. (Default: 200) + + --gibbs-number-of-samples + The total number of count vectors RSEM will collect from its Gibbs + samplers. (Default: 1000) + + --gibbs-sampling-gap + The number of rounds between two succinct count vectors RSEM + collects. If the count vector after round N is collected, the count + vector after round N + will also be collected. (Default: 1) + + --ci-credibility-level + The credibility level for credibility intervals. (Default: 0.95) + + --ci-memory + Maximum size (in memory, MB) of the auxiliary buffer used for + computing credibility intervals (CI). Set it larger for a faster CI + calculation. However, leaving 2 GB memory free for other usage is + recommended. (Default: 1024) + + --ci-number-of-samples-per-count-vector + The number of read generating probability vectors sampled per + sampled count vector. The crebility intervals are calculated by + first sampling P(C | D) and then sampling P(Theta | C) for each + sampled count vector. This option controls how many Theta vectors + are sampled per sampled count vector. (Default: 50) + + --samtools-sort-mem + Set the maximum memory per thread that can be used by 'samtools + sort'. represents the memory and accepts suffices 'K/M/G'. + RSEM will pass to the '-m' option of 'samtools sort'. + Please note that the default used here is different from the default + used by samtools. (Default: 1G) + + --keep-intermediate-files + Keep temporary files generated by RSEM. RSEM creates a temporary + directory, 'sample_name.temp', into which it puts all intermediate + output files. If this directory already exists, RSEM overwrites all + files generated by previous RSEM runs inside of it. By default, + after RSEM finishes, the temporary directory is deleted. Set this + option to prevent the deletion of this directory and the + intermediate files inside of it. (Default: off) + + --temporary-folder + Set where to put the temporary files generated by RSEM. If the + folder specified does not exist, RSEM will try to create it. + (Default: sample_name.temp) + + --time + Output time consumed by each step of RSEM to 'sample_name.time'. + (Default: off) + + DESCRIPTION + In its default mode, this program aligns input reads against a reference + transcriptome with Bowtie and calculates expression values using the + alignments. RSEM assumes the data are single-end reads with quality + scores, unless the '--paired-end' or '--no-qualities' options are + specified. Users may use an alternative aligner by specifying one of the + --sam and --bam options, and providing an alignment file in the + specified format. However, users should make sure that they align + against the indices generated by 'rsem-prepare-reference' and the + alignment file satisfies the requirements mentioned in ARGUMENTS + section. + + One simple way to make the alignment file satisfying RSEM's requirements + (assuming the aligner used put mates in a paired-end read adjacent) is + to use 'convert-sam-for-rsem' script. This script only accept SAM format + files as input. If a BAM format file is obtained, please use samtools to + convert it to a SAM file first. For example, if '/ref/mouse_125' is the + 'reference_name' and the SAM file is named 'input.sam', you can run the + following command: + + convert-sam-for-rsem /ref/mouse_125 input.sam -o input_for_rsem.sam + + For details, please refer to 'convert-sam-for-rsem's documentation page. + + The SAM/BAM format RSEM uses is v1.4. However, it is compatible with old + SAM/BAM format. However, RSEM cannot recognize 0x100 in the FLAG field. + In addition, RSEM requires SEQ and QUAL are not '*'. + + The user must run 'rsem-prepare-reference' with the appropriate + reference before using this program. + + For single-end data, it is strongly recommended that the user provide + the fragment length distribution parameters (--fragment-length-mean and + --fragment-length-sd). For paired-end data, RSEM will automatically + learn a fragment length distribution from the data. + + Please note that some of the default values for the Bowtie parameters + are not the same as those defined for Bowtie itself. + + The temporary directory and all intermediate files will be removed when + RSEM finishes unless '--keep-intermediate-files' is specified. + + With the '--calc-pme' option, posterior mean estimates will be + calculated in addition to maximum likelihood estimates. + + With the '--calc-ci' option, 95% credibility intervals and posterior + mean estimates will be calculated in addition to maximum likelihood + estimates. + + OUTPUT + sample_name.isoforms.results + File containing isoform level expression estimates. The first line + contains column names separated by the tab character. The format of + each line in the rest of this file is: + + transcript_id gene_id length effective_length expected_count TPM + FPKM IsoPct [posterior_mean_count + posterior_standard_deviation_of_count pme_TPM pme_FPKM + IsoPct_from_pme_TPM TPM_ci_lower_bound TPM_ci_upper_bound + FPKM_ci_lower_bound FPKM_ci_upper_bound] + + Fields are separated by the tab character. Fields within "[]" are + optional. They will not be presented if neither '--calc-pme' nor + '--calc-ci' is set. + + 'transcript_id' is the transcript name of this transcript. 'gene_id' + is the gene name of the gene which this transcript belongs to + (denote this gene as its parent gene). If no gene information is + provided, 'gene_id' and 'transcript_id' are the same. + + 'length' is this transcript's sequence length (poly(A) tail is not + counted). 'effective_length' counts only the positions that can + generate a valid fragment. If no poly(A) tail is added, + 'effective_length' is equal to transcript length - mean fragment + length + 1. If one transcript's effective length is less than 1, + this transcript's both effective length and abundance estimates are + set to 0. + + 'expected_count' is the sum of the posterior probability of each + read comes from this transcript over all reads. Because 1) each read + aligning to this transcript has a probability of being generated + from background noise; 2) RSEM may filter some alignable low quality + reads, the sum of expected counts for all transcript are generally + less than the total number of reads aligned. + + 'TPM' stands for Transcripts Per Million. It is a relative measure + of transcript abundance. The sum of all transcripts' TPM is 1 + million. 'FPKM' stands for Fragments Per Kilobase of transcript per + Million mapped reads. It is another relative measure of transcript + abundance. If we define l_bar be the mean transcript length in a + sample, which can be calculated as + + l_bar = \sum_i TPM_i / 10^6 * effective_length_i (i goes through + every transcript), + + the following equation is hold: + + FPKM_i = 10^3 / l_bar * TPM_i. + + We can see that the sum of FPKM is not a constant across samples. + + 'IsoPct' stands for isoform percentage. It is the percentage of this + transcript's abandunce over its parent gene's abandunce. If its + parent gene has only one isoform or the gene information is not + provided, this field will be set to 100. + + 'posterior_mean_count', 'pme_TPM', 'pme_FPKM' are posterior mean + estimates calculated by RSEM's Gibbs sampler. + 'posterior_standard_deviation_of_count' is the posterior standard + deviation of counts. 'IsoPct_from_pme_TPM' is the isoform percentage + calculated from 'pme_TPM' values. + + 'TPM_ci_lower_bound', 'TPM_ci_upper_bound', 'FPKM_ci_lower_bound' + and 'FPKM_ci_upper_bound' are lower(l) and upper(u) bounds of 95% + credibility intervals for TPM and FPKM values. The bounds are + inclusive (i.e. [l, u]). + + sample_name.genes.results + File containing gene level expression estimates. The first line + contains column names separated by the tab character. The format of + each line in the rest of this file is: + + gene_id transcript_id(s) length effective_length expected_count TPM + FPKM [posterior_mean_count posterior_standard_deviation_of_count + pme_TPM pme_FPKM TPM_ci_lower_bound TPM_ci_upper_bound + FPKM_ci_lower_bound FPKM_ci_upper_bound] + + Fields are separated by the tab character. Fields within "[]" are + optional. They will not be presented if neither '--calc-pme' nor + '--calc-ci' is set. + + 'transcript_id(s)' is a comma-separated list of transcript_ids + belonging to this gene. If no gene information is provided, + 'gene_id' and 'transcript_id(s)' are identical (the + 'transcript_id'). + + A gene's 'length' and 'effective_length' are defined as the weighted + average of its transcripts' lengths and effective lengths (weighted + by 'IsoPct'). A gene's abundance estimates are just the sum of its + transcripts' abundance estimates. + + sample_name.alleles.results + Only generated when the RSEM references are built with + allele-specific transcripts. + + This file contains allele level expression estimates for + allele-specific expression calculation. The first line contains + column names separated by the tab character. The format of each line + in the rest of this file is: + + allele_id transcript_id gene_id length effective_length + expected_count TPM FPKM AlleleIsoPct AlleleGenePct + [posterior_mean_count posterior_standard_deviation_of_count pme_TPM + pme_FPKM AlleleIsoPct_from_pme_TPM AlleleGenePct_from_pme_TPM + TPM_ci_lower_bound TPM_ci_upper_bound FPKM_ci_lower_bound + FPKM_ci_upper_bound] + + Fields are separated by the tab character. Fields within "[]" are + optional. They will not be presented if neither '--calc-pme' nor + '--calc-ci' is set. + + 'allele_id' is the allele-specific name of this allele-specific + transcript. + + 'AlleleIsoPct' stands for allele-specific percentage on isoform + level. It is the percentage of this allele-specific transcript's + abundance over its parent transcript's abundance. If its parent + transcript has only one allele variant form, this field will be set + to 100. + + 'AlleleGenePct' stands for allele-specific percentage on gene level. + It is the percentage of this allele-specific transcript's abundance + over its parent gene's abundance. + + 'AlleleIsoPct_from_pme_TPM' and 'AlleleGenePct_from_pme_TPM' have + similar meanings. They are calculated based on posterior mean + estimates. + + Please note that if this file is present, the fields 'length' and + 'effective_length' in 'sample_name.isoforms.results' should be + interpreted similarly as the corresponding definitions in + 'sample_name.genes.results'. + + sample_name.transcript.bam, sample_name.transcript.sorted.bam and + sample_name.transcript.sorted.bam.bai + Only generated when --no-bam-output is not specified. + + 'sample_name.transcript.bam' is a BAM-formatted file of read + alignments in transcript coordinates. The MAPQ field of each + alignment is set to min(100, floor(-10 * log10(1.0 - w) + 0.5)), + where w is the posterior probability of that alignment being the + true mapping of a read. In addition, RSEM pads a new tag ZW:f:value, + where value is a single precision floating number representing the + posterior probability. Because this file contains all alignment + lines produced by bowtie or user-specified aligners, it can also be + used as a replacement of the aligner generated BAM/SAM file. For + paired-end reads, if one mate has alignments but the other does not, + this file marks the alignable mate as "unmappable" (flag bit 0x4) + and appends an optional field "Z0:A:!". + + 'sample_name.transcript.sorted.bam' and + 'sample_name.transcript.sorted.bam.bai' are the sorted BAM file and + indices generated by samtools (included in RSEM package). + + sample_name.genome.bam, sample_name.genome.sorted.bam and + sample_name.genome.sorted.bam.bai + Only generated when --no-bam-output is not specified and + --output-genome-bam is specified. + + 'sample_name.genome.bam' is a BAM-formatted file of read alignments + in genomic coordinates. Alignments of reads that have identical + genomic coordinates (i.e., alignments to different isoforms that + share the same genomic region) are collapsed into one alignment. The + MAPQ field of each alignment is set to min(100, floor(-10 * + log10(1.0 - w) + 0.5)), where w is the posterior probability of that + alignment being the true mapping of a read. In addition, RSEM pads a + new tag ZW:f:value, where value is a single precision floating + number representing the posterior probability. If an alignment is + spliced, a XS:A:value tag is also added, where value is either '+' + or '-' indicating the strand of the transcript it aligns to. + + 'sample_name.genome.sorted.bam' and + 'sample_name.genome.sorted.bam.bai' are the sorted BAM file and + indices generated by samtools (included in RSEM package). + + sample_name.time + Only generated when --time is specified. + + It contains time (in seconds) consumed by aligning reads, estimating + expression levels and calculating credibility intervals. + + sample_name.stat + This is a folder instead of a file. All model related statistics are + stored in this folder. Use 'rsem-plot-model' can generate plots + using this folder. + + 'sample_name.stat/sample_name.cnt' contains alignment statistics. + The format and meanings of each field are described in + 'cnt_file_description.txt' under RSEM directory. + + 'sample_name.stat/sample_name.model' stores RNA-Seq model parameters + learned from the data. The format and meanings of each filed of this + file are described in 'model_file_description.txt' under RSEM + directory. + + EXAMPLES + Assume the path to the bowtie executables is in the user's PATH + environment variable. Reference files are under '/ref' with name + 'mouse_125'. + + 1) '/data/mmliver.fq', single-end reads with quality scores. Quality + scores are encoded as for 'GA pipeline version >= 1.3'. We want to use 8 + threads and generate a genome BAM file: + + rsem-calculate-expression --phred64-quals \ + -p 8 \ + --output-genome-bam \ + /data/mmliver.fq \ + /ref/mouse_125 \ + mmliver_single_quals + + 2) '/data/mmliver_1.fq' and '/data/mmliver_2.fq', paired-end reads with + quality scores. Quality scores are in SANGER format. We want to use 8 + threads and do not generate a genome BAM file: + + rsem-calculate-expression -p 8 \ + --paired-end \ + /data/mmliver_1.fq \ + /data/mmliver_2.fq \ + /ref/mouse_125 \ + mmliver_paired_end_quals + + 3) '/data/mmliver.fa', single-end reads without quality scores. We want + to use 8 threads: + + rsem-calculate-expression -p 8 \ + --no-qualities \ + /data/mmliver.fa \ + /ref/mouse_125 \ + mmliver_single_without_quals + + 4) Data are the same as 1). This time we assume the bowtie executables + are under '/sw/bowtie'. We want to take a fragment length distribution + into consideration. We set the fragment length mean to 150 and the + standard deviation to 35. In addition to a BAM file, we also want to + generate credibility intervals. We allow RSEM to use 1GB of memory for + CI calculation: + + rsem-calculate-expression --bowtie-path /sw/bowtie \ + --phred64-quals \ + --fragment-length-mean 150.0 \ + --fragment-length-sd 35.0 \ + -p 8 \ + --output-genome-bam \ + --calc-ci \ + --ci-memory 1024 \ + /data/mmliver.fq \ + /ref/mouse_125 \ + mmliver_single_quals + + 5) '/data/mmliver_paired_end_quals.bam', paired-end reads with quality + scores. We want to use 8 threads: + + rsem-calculate-expression --paired-end \ + --bam \ + -p 8 \ + /data/mmliver_paired_end_quals.bam \ + /ref/mouse_125 \ + mmliver_paired_end_quals + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpcqum6ex8/86ec48a5-8735-4696-941d-8022971049f6:/data', '--log-driver=none', 'aarjunrao/rsem:1.2.20', '--paired-end', '-p', '80.0', '--bam', '/data/star_transcriptome.bam', '--no-bam-output', '/data/rsem_indexes/hg38', 'rsem']' returned non-zero exit status 2. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/expression_profiling/rsem.py", line 88, in run_rsem + dockerhub=univ_options['dockerhub'], tool_version=rsem_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (2)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpcqum6ex8/86ec48a5-8735-4696-941d-8022971049f6:/data --log-driver=none aarjunrao/rsem:1.2.20 --paired-end -p 80.0 --bam /data/star_transcriptome.bam --no-bam-output /data/rsem_indexes/hg38 rsem" + [2020-11-17T14:45:33-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-17T14:45:34-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with ID kind-run_rsem/instance-mzzok2e3 to 0 +[2020-11-17T14:45:34-0800] [MainThread] [W] [toil.leader] Job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with ID kind-run_rsem/instance-mzzok2e3 is completely failed +[2020-11-17T15:13:22-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-17T16:13:23-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-17T17:13:23-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-17T18:13:24-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-17T19:13:25-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-17T20:13:25-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-17T21:13:26-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-17T22:13:26-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-17T23:13:27-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T00:13:27-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T01:13:28-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T02:13:28-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T03:13:29-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T04:13:29-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T05:13:30-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T06:13:30-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T07:13:31-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T08:13:31-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T09:13:31-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T10:13:32-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T11:13:32-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T12:13:33-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T13:13:33-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T14:13:34-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T15:13:35-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T16:13:35-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T17:13:36-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T18:13:36-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T19:13:37-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T20:13:37-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T21:13:38-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T22:13:38-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-18T23:13:39-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T00:13:39-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T01:13:40-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T02:13:40-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T03:13:41-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T04:13:41-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T05:13:42-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T06:13:42-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T07:13:42-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T08:13:43-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T09:13:43-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T10:13:44-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T11:13:44-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T12:13:45-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T13:13:45-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T14:13:46-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T15:13:47-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T16:13:47-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T17:13:48-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T18:13:48-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T19:13:49-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T20:13:49-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T21:13:50-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T22:13:50-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-19T23:13:51-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-20T00:13:51-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-20T01:13:52-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-20T02:13:52-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-20T03:13:53-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-20T04:13:53-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-20T05:13:54-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-20T06:13:54-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-20T07:13:55-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-20T08:13:55-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-20T09:13:56-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-20T10:13:57-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-20T11:13:57-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-20T12:13:58-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-20T13:13:58-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-20T14:13:59-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-20T15:13:59-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-20T16:14:00-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-20T17:14:00-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run diff --git a/errors/2020-11-24-00-22-56.txt b/errors/2020-11-24-00-22-56.txt new file mode 100644 index 00000000..73accb29 --- /dev/null +++ b/errors/2020-11-24-00-22-56.txt @@ -0,0 +1,168 @@ +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-q4vn5u6_ +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ovr5_kcb +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-jkvqxjgf +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xbtj2suv +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-etqwe7dv +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-bv9m7hf3 +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ewrzdiri +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-eg7xdcao +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-06rjxk27 +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-klw60gaa +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-7_6wjfvk +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xz7ir9sh +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-k3tx4a2p +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-mppqh5vo +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-d9qjg3jl +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_rsem/instance-mzzok2e3 +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ge6vacd7 +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-2dtgopje +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-aux6_f19 +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r7m8gybn +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-9uka9hrw +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-c60ojvxr +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r1uv5wsv +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-v628y3w0 +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-2u48n_5f +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-rd57kqhd +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-0xdl6glf +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-yb2yf65u +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_fusion/instance-g9_082sr +[2020-11-24T00:23:07-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-uuibculn +[2020-11-24T00:23:07-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-klw60gaa with job batch system ID: 0 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-c60ojvxr with job batch system ID: 1 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-uuibculn with job batch system ID: 2 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-aux6_f19 with job batch system ID: 3 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-d9qjg3jl with job batch system ID: 4 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-eg7xdcao with job batch system ID: 5 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mppqh5vo with job batch system ID: 6 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yb2yf65u with job batch system ID: 7 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-q4vn5u6_ with job batch system ID: 8 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ge6vacd7 with job batch system ID: 9 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-7_6wjfvk with job batch system ID: 10 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2u48n_5f with job batch system ID: 11 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jkvqxjgf with job batch system ID: 12 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bv9m7hf3 with job batch system ID: 13 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r1uv5wsv with job batch system ID: 14 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9uka9hrw with job batch system ID: 15 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ewrzdiri with job batch system ID: 16 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ovr5_kcb with job batch system ID: 17 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xbtj2suv with job batch system ID: 18 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0xdl6glf with job batch system ID: 19 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-v628y3w0 with job batch system ID: 20 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-k3tx4a2p with job batch system ID: 21 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-06rjxk27 with job batch system ID: 22 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-rd57kqhd with job batch system ID: 23 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xz7ir9sh with job batch system ID: 24 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r7m8gybn with job batch system ID: 25 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2dtgopje with job batch system ID: 26 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-etqwe7dv with job batch system ID: 27 and cores: 80, disk: 8.2 G, and memory: 2.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_fusion' kind-run_fusion/instance-g9_082sr with job batch system ID: 28 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with job batch system ID: 29 and cores: 80, disk: 5.0 G, and memory: 2.0 G +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphhbv9ijs/worker_log.txt +[2020-11-24T00:23:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8h862v2o/worker_log.txt +[2020-11-24T00:23:10-0800] [MainThread] [I] [toil.leader] 2 jobs are running, 28 jobs are issued and waiting to run +[2020-11-24T00:24:25-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-klw60gaa. +[2020-11-24T00:24:25-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-klw60gaa +[2020-11-24T00:24:25-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-klw60gaa +[2020-11-24T00:24:25-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-klw60gaa follows: +=========> + [2020-11-24T00:23:08-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-24T00:23:08-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 59, in + nthreads=int(val) + ValueError: invalid literal for int() with base 10: '80.0' + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8h862v2o/68681bd8-a89b-4709-ac30-37eb251d0756:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '80.0']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8h862v2o/68681bd8-a89b-4709-ac30-37eb251d0756:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_dna -e /home/phlat-1.0 -o /data -p 80.0" + [2020-11-24T00:24:24-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8h862v2o/68681bd8-a89b-4709-ac30-37eb251d0756/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8h862v2o/68681bd8-a89b-4709-ac30-37eb251d0756/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8h862v2o/68681bd8-a89b-4709-ac30-37eb251d0756/phlat_index.tar.gz'} +<========= +[2020-11-24T00:24:25-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-klw60gaa with ID kind-run_phlat/instance-klw60gaa to 1 +[2020-11-24T00:24:25-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-klw60gaa with job batch system ID: 30 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-24T00:24:25-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-c60ojvxr. +[2020-11-24T00:24:25-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-c60ojvxr +[2020-11-24T00:24:25-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-c60ojvxr +[2020-11-24T00:24:25-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-c60ojvxr follows: +=========> + [2020-11-24T00:23:08-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-24T00:23:08-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 59, in + nthreads=int(val) + ValueError: invalid literal for int() with base 10: '80.0' + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphhbv9ijs/2afd3379-62d5-4264-b1fe-d5398674c450:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'normal_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '80.0']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphhbv9ijs/2afd3379-62d5-4264-b1fe-d5398674c450:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag normal_dna -e /home/phlat-1.0 -o /data -p 80.0" + [2020-11-24T00:24:24-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphhbv9ijs/2afd3379-62d5-4264-b1fe-d5398674c450/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphhbv9ijs/2afd3379-62d5-4264-b1fe-d5398674c450/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphhbv9ijs/2afd3379-62d5-4264-b1fe-d5398674c450/phlat_index.tar.gz'} +<========= +[2020-11-24T00:24:25-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-c60ojvxr with ID kind-run_phlat/instance-c60ojvxr to 1 +[2020-11-24T00:24:25-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-c60ojvxr with job batch system ID: 31 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpa32umob0/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpnqaotg9k/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpsjam80eq/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpj6sb079c/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpu4bqujtv/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpnx1682f0/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpifsw2agc/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpgspcs4cz/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp07a2l451/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpftjb07hi/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmptsbo38dy/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpu8okxq9u/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfbc1tdh2/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpc9r11_a3/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpiarnftdo/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpu3dvnh0m/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpy9178f3e/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpiz21171d/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpcp5s3kwz/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpr0dr6hxe/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpzf41rmto/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpkq0c2d43/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9kfy2ffr/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpi3_3k80d/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpsk3_6s9i/worker_log.txt +[2020-11-24T00:24:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpq6qxzo2b/worker_log.txt diff --git a/errors/2020-11-24-00-26-36.txt b/errors/2020-11-24-00-26-36.txt new file mode 100644 index 00000000..10d348cb --- /dev/null +++ b/errors/2020-11-24-00-26-36.txt @@ -0,0 +1,17 @@ +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/bin/ProTECT", line 11, in + load_entry_point('protect', 'console_scripts', 'ProTECT')() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/pkg_resources/__init__.py", line 565, in load_entry_point + return get_distribution(dist).load_entry_point(group, name) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/pkg_resources/__init__.py", line 2631, in load_entry_point + return ep.load() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/pkg_resources/__init__.py", line 2291, in load + return self.resolve() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/pkg_resources/__init__.py", line 2297, in resolve + module = __import__(self.module_name, fromlist=['__name__'], level=0) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 46, in + from protect.expression_profiling.rsem import wrap_rsem + File "/private/home/drkthomp/protect3/src/protect/expression_profiling/rsem.py", line 81 + '-p', 10), + ^ +SyntaxError: invalid syntax diff --git a/errors/2020-11-24-00-27-18.txt b/errors/2020-11-24-00-27-18.txt new file mode 100644 index 00000000..05486d5a --- /dev/null +++ b/errors/2020-11-24-00-27-18.txt @@ -0,0 +1,17 @@ +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/bin/ProTECT", line 11, in + load_entry_point('protect', 'console_scripts', 'ProTECT')() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/pkg_resources/__init__.py", line 565, in load_entry_point + return get_distribution(dist).load_entry_point(group, name) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/pkg_resources/__init__.py", line 2631, in load_entry_point + return ep.load() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/pkg_resources/__init__.py", line 2291, in load + return self.resolve() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/pkg_resources/__init__.py", line 2297, in resolve + module = __import__(self.module_name, fromlist=['__name__'], level=0) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 46, in + from protect.expression_profiling.rsem import wrap_rsem + File "/private/home/drkthomp/protect3/src/protect/expression_profiling/rsem.py", line 81 + '-p', '10'), + ^ +SyntaxError: invalid syntax diff --git a/errors/2020-11-24-00-28-09.txt b/errors/2020-11-24-00-28-09.txt new file mode 100644 index 00000000..12634964 --- /dev/null +++ b/errors/2020-11-24-00-28-09.txt @@ -0,0 +1,166 @@ +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-v628y3w0 +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xbtj2suv +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-uuibculn +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-jkvqxjgf +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ovr5_kcb +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-d9qjg3jl +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_fusion/instance-g9_082sr +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ewrzdiri +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-klw60gaa +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-eg7xdcao +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ge6vacd7 +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r1uv5wsv +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-7_6wjfvk +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-9uka9hrw +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-0xdl6glf +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-mppqh5vo +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-bv9m7hf3 +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-06rjxk27 +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-q4vn5u6_ +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-yb2yf65u +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-2dtgopje +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-aux6_f19 +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-c60ojvxr +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xz7ir9sh +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-rd57kqhd +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r7m8gybn +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-2u48n_5f +[2020-11-24T00:28:10-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-k3tx4a2p +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-klw60gaa with job batch system ID: 0 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-c60ojvxr with job batch system ID: 1 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-uuibculn with job batch system ID: 2 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-aux6_f19 with job batch system ID: 3 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-d9qjg3jl with job batch system ID: 4 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-eg7xdcao with job batch system ID: 5 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mppqh5vo with job batch system ID: 6 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yb2yf65u with job batch system ID: 7 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-q4vn5u6_ with job batch system ID: 8 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ge6vacd7 with job batch system ID: 9 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-7_6wjfvk with job batch system ID: 10 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2u48n_5f with job batch system ID: 11 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jkvqxjgf with job batch system ID: 12 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bv9m7hf3 with job batch system ID: 13 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r1uv5wsv with job batch system ID: 14 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9uka9hrw with job batch system ID: 15 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ewrzdiri with job batch system ID: 16 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ovr5_kcb with job batch system ID: 17 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xbtj2suv with job batch system ID: 18 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0xdl6glf with job batch system ID: 19 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-v628y3w0 with job batch system ID: 20 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-k3tx4a2p with job batch system ID: 21 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-06rjxk27 with job batch system ID: 22 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-rd57kqhd with job batch system ID: 23 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xz7ir9sh with job batch system ID: 24 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r7m8gybn with job batch system ID: 25 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2dtgopje with job batch system ID: 26 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-etqwe7dv with job batch system ID: 27 and cores: 80, disk: 8.2 G, and memory: 2.0 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_fusion' kind-run_fusion/instance-g9_082sr with job batch system ID: 28 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-24T00:28:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with job batch system ID: 29 and cores: 80, disk: 5.0 G, and memory: 2.0 G +[2020-11-24T00:28:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4zmzxj8y/worker_log.txt +[2020-11-24T00:28:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvrcvpnr6/worker_log.txt +[2020-11-24T00:28:12-0800] [MainThread] [I] [toil.leader] 2 jobs are running, 28 jobs are issued and waiting to run +[2020-11-24T00:29:23-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-c60ojvxr. +[2020-11-24T00:29:23-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-c60ojvxr +[2020-11-24T00:29:23-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-c60ojvxr +[2020-11-24T00:29:23-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-c60ojvxr follows: +=========> + [2020-11-24T00:28:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-24T00:28:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 59, in + nthreads=int(val) + ValueError: invalid literal for int() with base 10: '80.0' + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4zmzxj8y/25f798dd-2b69-43ed-bfda-259a1b536a08:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'normal_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '80.0']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4zmzxj8y/25f798dd-2b69-43ed-bfda-259a1b536a08:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag normal_dna -e /home/phlat-1.0 -o /data -p 80.0" + [2020-11-24T00:29:22-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4zmzxj8y/25f798dd-2b69-43ed-bfda-259a1b536a08/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4zmzxj8y/25f798dd-2b69-43ed-bfda-259a1b536a08/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4zmzxj8y/25f798dd-2b69-43ed-bfda-259a1b536a08/phlat_index.tar.gz'} +<========= +[2020-11-24T00:29:23-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-c60ojvxr with ID kind-run_phlat/instance-c60ojvxr to 1 +[2020-11-24T00:29:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-c60ojvxr with job batch system ID: 30 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmprrdwrws0/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmprh0na5lj/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpkr6lagkr/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpyoe54c_a/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpozjhdx7r/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpx3be5sct/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpi0s7sgj7/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpogi1pavb/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphit_ea9i/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpdddscste/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpbqusqiy9/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp6rg2x2ug/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp2m2rgnfe/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjmum9gnl/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp6hwnwp1_/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpe97_1mrx/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfdz9301x/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpateqrdbl/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp5cbk_3re/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmppkpkl84l/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_2ut_2ae/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpeqeobupo/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpp6opgunz/worker_log.txt +[2020-11-24T00:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpzl_3htl6/worker_log.txt +[2020-11-24T00:29:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_gx6i0ti/worker_log.txt +[2020-11-24T00:29:25-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-klw60gaa. +[2020-11-24T00:29:25-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-klw60gaa +[2020-11-24T00:29:25-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-klw60gaa +[2020-11-24T00:29:25-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-klw60gaa follows: +=========> + [2020-11-24T00:28:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-24T00:28:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 59, in + nthreads=int(val) + ValueError: invalid literal for int() with base 10: '80.0' + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvrcvpnr6/ec346eaf-890e-4fc6-80c2-ad0f868a5a8b:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '-tag', 'tumor_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '80.0']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvrcvpnr6/ec346eaf-890e-4fc6-80c2-ad0f868a5a8b:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tumor_dna -e /home/phlat-1.0 -o /data -p 80.0" + [2020-11-24T00:29:24-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvrcvpnr6/ec346eaf-890e-4fc6-80c2-ad0f868a5a8b/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvrcvpnr6/ec346eaf-890e-4fc6-80c2-ad0f868a5a8b/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvrcvpnr6/ec346eaf-890e-4fc6-80c2-ad0f868a5a8b/phlat_index.tar.gz'} +<========= +[2020-11-24T00:29:25-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-klw60gaa with ID kind-run_phlat/instance-klw60gaa to 1 +[2020-11-24T00:29:25-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-klw60gaa with job batch system ID: 31 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-24T00:29:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmph1o9v94d/worker_log.txt diff --git a/errors/2020-11-24-00-47-40.txt b/errors/2020-11-24-00-47-40.txt new file mode 100644 index 00000000..58191d0a --- /dev/null +++ b/errors/2020-11-24-00-47-40.txt @@ -0,0 +1,62 @@ +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-klw60gaa +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ovr5_kcb +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ewrzdiri +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ge6vacd7 +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xz7ir9sh +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-0xdl6glf +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-q4vn5u6_ +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-2dtgopje +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_rsem/instance-mzzok2e3 +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-06rjxk27 +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-d9qjg3jl +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-aux6_f19 +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-bv9m7hf3 +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-k3tx4a2p +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-v628y3w0 +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-uuibculn +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r1uv5wsv +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xbtj2suv +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-2u48n_5f +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-mppqh5vo +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-9uka9hrw +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-eg7xdcao +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r7m8gybn +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-yb2yf65u +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-c60ojvxr +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-rd57kqhd +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-7_6wjfvk +[2020-11-24T00:47:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-jkvqxjgf +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-klw60gaa with job batch system ID: 0 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-c60ojvxr with job batch system ID: 1 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-uuibculn with job batch system ID: 2 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-aux6_f19 with job batch system ID: 3 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-d9qjg3jl with job batch system ID: 4 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-eg7xdcao with job batch system ID: 5 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mppqh5vo with job batch system ID: 6 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yb2yf65u with job batch system ID: 7 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-q4vn5u6_ with job batch system ID: 8 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ge6vacd7 with job batch system ID: 9 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-7_6wjfvk with job batch system ID: 10 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2u48n_5f with job batch system ID: 11 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jkvqxjgf with job batch system ID: 12 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bv9m7hf3 with job batch system ID: 13 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r1uv5wsv with job batch system ID: 14 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9uka9hrw with job batch system ID: 15 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ewrzdiri with job batch system ID: 16 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ovr5_kcb with job batch system ID: 17 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xbtj2suv with job batch system ID: 18 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0xdl6glf with job batch system ID: 19 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-v628y3w0 with job batch system ID: 20 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-k3tx4a2p with job batch system ID: 21 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-06rjxk27 with job batch system ID: 22 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-rd57kqhd with job batch system ID: 23 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xz7ir9sh with job batch system ID: 24 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r7m8gybn with job batch system ID: 25 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2dtgopje with job batch system ID: 26 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-etqwe7dv with job batch system ID: 27 and cores: 80, disk: 8.2 G, and memory: 2.0 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_fusion' kind-run_fusion/instance-g9_082sr with job batch system ID: 28 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-24T00:47:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with job batch system ID: 29 and cores: 80, disk: 5.0 G, and memory: 2.0 G +[2020-11-24T00:47:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmps5ymxacm/worker_log.txt +[2020-11-24T00:47:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpi0ix8uru/worker_log.txt +[2020-11-24T00:47:44-0800] [MainThread] [I] [toil.leader] 2 jobs are running, 28 jobs are issued and waiting to run diff --git a/errors/2020-11-24-00-52-41.txt b/errors/2020-11-24-00-52-41.txt new file mode 100644 index 00000000..6f9c6513 --- /dev/null +++ b/errors/2020-11-24-00-52-41.txt @@ -0,0 +1,130 @@ +[2020-11-24T00:52:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-klw60gaa +[2020-11-24T00:52:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-c60ojvxr +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-klw60gaa with job batch system ID: 0 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-c60ojvxr with job batch system ID: 1 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-uuibculn with job batch system ID: 2 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-aux6_f19 with job batch system ID: 3 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-d9qjg3jl with job batch system ID: 4 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-eg7xdcao with job batch system ID: 5 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mppqh5vo with job batch system ID: 6 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yb2yf65u with job batch system ID: 7 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-q4vn5u6_ with job batch system ID: 8 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ge6vacd7 with job batch system ID: 9 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-7_6wjfvk with job batch system ID: 10 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2u48n_5f with job batch system ID: 11 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jkvqxjgf with job batch system ID: 12 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bv9m7hf3 with job batch system ID: 13 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r1uv5wsv with job batch system ID: 14 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9uka9hrw with job batch system ID: 15 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ewrzdiri with job batch system ID: 16 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ovr5_kcb with job batch system ID: 17 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xbtj2suv with job batch system ID: 18 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0xdl6glf with job batch system ID: 19 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-v628y3w0 with job batch system ID: 20 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-k3tx4a2p with job batch system ID: 21 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-06rjxk27 with job batch system ID: 22 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-rd57kqhd with job batch system ID: 23 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xz7ir9sh with job batch system ID: 24 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r7m8gybn with job batch system ID: 25 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2dtgopje with job batch system ID: 26 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-etqwe7dv with job batch system ID: 27 and cores: 80, disk: 8.2 G, and memory: 2.0 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_fusion' kind-run_fusion/instance-g9_082sr with job batch system ID: 28 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-24T00:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with job batch system ID: 29 and cores: 80, disk: 5.0 G, and memory: 2.0 G +[2020-11-24T00:52:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpx775xbz9/worker_log.txt +[2020-11-24T00:52:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpu554trl4/worker_log.txt +[2020-11-24T00:52:45-0800] [MainThread] [I] [toil.leader] 2 jobs are running, 28 jobs are issued and waiting to run +[2020-11-24T00:53:52-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-klw60gaa. +[2020-11-24T00:53:52-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-klw60gaa +[2020-11-24T00:53:52-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-klw60gaa +[2020-11-24T00:53:52-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-klw60gaa follows: +=========> + [2020-11-24T00:52:44-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-24T00:52:44-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 286, in check_call + retcode = call(*popenargs, **kwargs) + File "/usr/lib64/python3.6/subprocess.py", line 267, in call + with Popen(*popenargs, **kwargs) as p: + File "/usr/lib64/python3.6/subprocess.py", line 709, in __init__ + restore_signals, start_new_session) + File "/usr/lib64/python3.6/subprocess.py", line 1275, in _execute_child + restore_signals, start_new_session, preexec_fn) + TypeError: expected str, bytes or os.PathLike object, not float + [2020-11-24T00:53:51-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpx775xbz9/d85cccee-3cd6-47d4-ab30-4e74a6e9892a/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpx775xbz9/d85cccee-3cd6-47d4-ab30-4e74a6e9892a/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpx775xbz9/d85cccee-3cd6-47d4-ab30-4e74a6e9892a/phlat_index.tar.gz'} +<========= +[2020-11-24T00:53:52-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-klw60gaa with ID kind-run_phlat/instance-klw60gaa to 1 +[2020-11-24T00:53:52-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-klw60gaa with job batch system ID: 30 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-24T00:53:52-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-c60ojvxr. +[2020-11-24T00:53:52-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-c60ojvxr +[2020-11-24T00:53:52-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-c60ojvxr +[2020-11-24T00:53:52-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-c60ojvxr follows: +=========> + [2020-11-24T00:52:44-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-24T00:52:44-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 286, in check_call + retcode = call(*popenargs, **kwargs) + File "/usr/lib64/python3.6/subprocess.py", line 267, in call + with Popen(*popenargs, **kwargs) as p: + File "/usr/lib64/python3.6/subprocess.py", line 709, in __init__ + restore_signals, start_new_session) + File "/usr/lib64/python3.6/subprocess.py", line 1275, in _execute_child + restore_signals, start_new_session, preexec_fn) + TypeError: expected str, bytes or os.PathLike object, not float + [2020-11-24T00:53:51-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpu554trl4/2e141dd0-1756-496d-a4e1-0ac3ea3dd974/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpu554trl4/2e141dd0-1756-496d-a4e1-0ac3ea3dd974/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpu554trl4/2e141dd0-1756-496d-a4e1-0ac3ea3dd974/phlat_index.tar.gz'} +<========= +[2020-11-24T00:53:52-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-c60ojvxr with ID kind-run_phlat/instance-c60ojvxr to 1 +[2020-11-24T00:53:52-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-c60ojvxr with job batch system ID: 31 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp6ft2lpqy/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3cj7r8td/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp5_sjewg1/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpqczzpf8c/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp2730h51r/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpwxp25x3d/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpb8m07v31/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpyvolvn_t/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmptf20a531/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpg0k_yfqq/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp65bnrttn/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpf4xwitup/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp6cf9brs0/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpf3hfc7an/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpyzstbwa0/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpkcflk7cg/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpbnwofdmf/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpn2efv2g5/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmps2t7ix55/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphyskm4c1/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpchsv6u14/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp1es6by6z/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpv8hckx7y/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmptjs8b_vd/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpedugtriz/worker_log.txt +[2020-11-24T00:53:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_6h9cq3w/worker_log.txt diff --git a/errors/2020-11-24-00-54-50.txt b/errors/2020-11-24-00-54-50.txt new file mode 100644 index 00000000..f28c298b --- /dev/null +++ b/errors/2020-11-24-00-54-50.txt @@ -0,0 +1,130 @@ +[2020-11-24T00:54:51-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-klw60gaa +[2020-11-24T00:54:51-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-c60ojvxr +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-klw60gaa with job batch system ID: 0 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-c60ojvxr with job batch system ID: 1 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-uuibculn with job batch system ID: 2 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-aux6_f19 with job batch system ID: 3 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-d9qjg3jl with job batch system ID: 4 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-eg7xdcao with job batch system ID: 5 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mppqh5vo with job batch system ID: 6 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yb2yf65u with job batch system ID: 7 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-q4vn5u6_ with job batch system ID: 8 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ge6vacd7 with job batch system ID: 9 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-7_6wjfvk with job batch system ID: 10 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2u48n_5f with job batch system ID: 11 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jkvqxjgf with job batch system ID: 12 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bv9m7hf3 with job batch system ID: 13 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r1uv5wsv with job batch system ID: 14 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9uka9hrw with job batch system ID: 15 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ewrzdiri with job batch system ID: 16 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ovr5_kcb with job batch system ID: 17 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xbtj2suv with job batch system ID: 18 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0xdl6glf with job batch system ID: 19 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-v628y3w0 with job batch system ID: 20 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-k3tx4a2p with job batch system ID: 21 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-06rjxk27 with job batch system ID: 22 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-rd57kqhd with job batch system ID: 23 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xz7ir9sh with job batch system ID: 24 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r7m8gybn with job batch system ID: 25 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2dtgopje with job batch system ID: 26 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-etqwe7dv with job batch system ID: 27 and cores: 80, disk: 8.2 G, and memory: 2.0 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_fusion' kind-run_fusion/instance-g9_082sr with job batch system ID: 28 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-24T00:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with job batch system ID: 29 and cores: 80, disk: 5.0 G, and memory: 2.0 G +[2020-11-24T00:54:52-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpkj9kumdi/worker_log.txt +[2020-11-24T00:54:52-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpu8sdy96w/worker_log.txt +[2020-11-24T00:54:53-0800] [MainThread] [I] [toil.leader] 2 jobs are running, 28 jobs are issued and waiting to run +[2020-11-24T00:56:00-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-c60ojvxr. +[2020-11-24T00:56:00-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-c60ojvxr +[2020-11-24T00:56:00-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-c60ojvxr +[2020-11-24T00:56:00-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-c60ojvxr follows: +=========> + [2020-11-24T00:54:52-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-24T00:54:52-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 286, in check_call + retcode = call(*popenargs, **kwargs) + File "/usr/lib64/python3.6/subprocess.py", line 267, in call + with Popen(*popenargs, **kwargs) as p: + File "/usr/lib64/python3.6/subprocess.py", line 709, in __init__ + restore_signals, start_new_session) + File "/usr/lib64/python3.6/subprocess.py", line 1275, in _execute_child + restore_signals, start_new_session, preexec_fn) + TypeError: expected str, bytes or os.PathLike object, not int + [2020-11-24T00:55:59-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpkj9kumdi/34216d81-b330-421a-b9ab-6144134abb32/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpkj9kumdi/34216d81-b330-421a-b9ab-6144134abb32/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpkj9kumdi/34216d81-b330-421a-b9ab-6144134abb32/phlat_index.tar.gz'} +<========= +[2020-11-24T00:56:00-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-c60ojvxr with ID kind-run_phlat/instance-c60ojvxr to 1 +[2020-11-24T00:56:00-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-c60ojvxr with job batch system ID: 30 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-24T00:56:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjzp13p4n/worker_log.txt +[2020-11-24T00:56:01-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-klw60gaa. +[2020-11-24T00:56:01-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-klw60gaa +[2020-11-24T00:56:01-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-klw60gaa +[2020-11-24T00:56:01-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-klw60gaa follows: +=========> + [2020-11-24T00:54:52-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-24T00:54:52-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 286, in check_call + retcode = call(*popenargs, **kwargs) + File "/usr/lib64/python3.6/subprocess.py", line 267, in call + with Popen(*popenargs, **kwargs) as p: + File "/usr/lib64/python3.6/subprocess.py", line 709, in __init__ + restore_signals, start_new_session) + File "/usr/lib64/python3.6/subprocess.py", line 1275, in _execute_child + restore_signals, start_new_session, preexec_fn) + TypeError: expected str, bytes or os.PathLike object, not int + [2020-11-24T00:56:00-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpu8sdy96w/b3d3e255-da9f-4bd8-a32a-a9e69329d08b/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpu8sdy96w/b3d3e255-da9f-4bd8-a32a-a9e69329d08b/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpu8sdy96w/b3d3e255-da9f-4bd8-a32a-a9e69329d08b/phlat_index.tar.gz'} +<========= +[2020-11-24T00:56:01-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-klw60gaa with ID kind-run_phlat/instance-klw60gaa to 1 +[2020-11-24T00:56:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-klw60gaa with job batch system ID: 31 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpn6ches9i/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpe9be_llf/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmptnsx72h4/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpbr0peewp/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_iy9gf48/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpj7ztpa08/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvp8dl72p/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp08ksg85i/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpljmyj7vj/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpz181a7n3/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0uey4rwg/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp29w4z_5j/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpaqt7dzht/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp5mqa5xbm/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9kstwdug/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp236_fd6d/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpxtbaic0r/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp1oqyfuol/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjku6h034/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmplrm7bwse/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmptclgp_yv/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpwsdlunhg/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9klmdnn9/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpbl8vepqh/worker_log.txt +[2020-11-24T00:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpgimu2pmy/worker_log.txt diff --git a/errors/2020-11-24-00-56-41.txt b/errors/2020-11-24-00-56-41.txt new file mode 100644 index 00000000..e19ecd7c --- /dev/null +++ b/errors/2020-11-24-00-56-41.txt @@ -0,0 +1,266 @@ +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-d9qjg3jl +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xz7ir9sh +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-9uka9hrw +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-rd57kqhd +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ge6vacd7 +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-7_6wjfvk +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-yb2yf65u +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-2u48n_5f +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xbtj2suv +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-v628y3w0 +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-c60ojvxr +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-2dtgopje +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-mppqh5vo +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-etqwe7dv +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r1uv5wsv +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-klw60gaa +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-k3tx4a2p +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r7m8gybn +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-uuibculn +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-eg7xdcao +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-q4vn5u6_ +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-0xdl6glf +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-jkvqxjgf +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-06rjxk27 +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-aux6_f19 +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ewrzdiri +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ovr5_kcb +[2020-11-24T00:56:43-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-bv9m7hf3 +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-klw60gaa with job batch system ID: 0 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-c60ojvxr with job batch system ID: 1 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-uuibculn with job batch system ID: 2 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-aux6_f19 with job batch system ID: 3 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-d9qjg3jl with job batch system ID: 4 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-eg7xdcao with job batch system ID: 5 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mppqh5vo with job batch system ID: 6 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yb2yf65u with job batch system ID: 7 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-q4vn5u6_ with job batch system ID: 8 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ge6vacd7 with job batch system ID: 9 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-7_6wjfvk with job batch system ID: 10 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2u48n_5f with job batch system ID: 11 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jkvqxjgf with job batch system ID: 12 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bv9m7hf3 with job batch system ID: 13 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r1uv5wsv with job batch system ID: 14 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9uka9hrw with job batch system ID: 15 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ewrzdiri with job batch system ID: 16 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ovr5_kcb with job batch system ID: 17 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xbtj2suv with job batch system ID: 18 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0xdl6glf with job batch system ID: 19 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-v628y3w0 with job batch system ID: 20 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-k3tx4a2p with job batch system ID: 21 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-06rjxk27 with job batch system ID: 22 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-rd57kqhd with job batch system ID: 23 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xz7ir9sh with job batch system ID: 24 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r7m8gybn with job batch system ID: 25 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2dtgopje with job batch system ID: 26 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-etqwe7dv with job batch system ID: 27 and cores: 80, disk: 8.2 G, and memory: 2.0 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_fusion' kind-run_fusion/instance-g9_082sr with job batch system ID: 28 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-24T00:56:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with job batch system ID: 29 and cores: 80, disk: 5.0 G, and memory: 2.0 G +[2020-11-24T00:56:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvb41yyc4/worker_log.txt +[2020-11-24T00:56:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphx0tgt_y/worker_log.txt +[2020-11-24T00:56:45-0800] [MainThread] [I] [toil.leader] 2 jobs are running, 28 jobs are issued and waiting to run +[2020-11-24T01:03:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-24-2020 01:03:39: Ran phlat on TEST:normal_dna successfully +[2020-11-24T01:03:40-0800] [MainThread] [I] [toil.leader] Job ended: 'run_phlat' kind-run_phlat/instance-c60ojvxr +[2020-11-24T01:03:40-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-wyv5bfui with job batch system ID: 30 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpzhshrqvs/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmptsdjn27x/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpa0zjnmcb/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpkjn29toh/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpppmdrs5y/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmps33zvev4/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpgwk1uvmf/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp60r4a80g/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3g_3rhp1/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpzzhh_rnv/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjlxmok16/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpd4etjbgq/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp639f2b0q/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjhaq7eot/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp5pne01ho/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpexod06mb/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpc3cq1va_/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmps1jbx6gn/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpw5fnjxh0/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpvqmh76wu/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpd30398_j/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp5hqteo0x/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpddsu00ob/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpqdq_ps5_/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmps9fjspj4/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpn1ifbw1i/worker_log.txt +[2020-11-24T01:03:42-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-wyv5bfui +[2020-11-24T01:05:29-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-24-2020 01:05:29: Ran phlat on TEST:tumor_dna successfully +[2020-11-24T01:05:30-0800] [MainThread] [I] [toil.leader] Job ended: 'run_phlat' kind-run_phlat/instance-klw60gaa +[2020-11-24T01:05:30-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-m1isjmol with job batch system ID: 31 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-24T01:05:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpaymcsw14/worker_log.txt +[2020-11-24T01:05:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpiv_6rqgc/worker_log.txt +[2020-11-24T01:05:31-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-m1isjmol +[2020-11-24T01:29:11-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_fusion file:/scratch/drkthomp/jobStore kind-run_fusion/instance-g9_082sr. +[2020-11-24T01:29:11-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-24T01:29:11-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-24T01:29:11-0800] [MainThread] [W] [toil.leader] Log from job kind-run_fusion/instance-g9_082sr follows: +=========> + [2020-11-24T01:05:30-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-24T01:05:30-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Error, don't understand arguments: [--CPU 80.0] at /opt/STAR-Fusion-v1.0.0/STAR-Fusion line 167. + [2020-11-24T01:29:01-0800] [MainThread] [W] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-v67rsjos/cleanup/file-d_nkrp7o/stream used 111.60% (59.6 GB [64023429120B] used, 53.4 GB [57369019908B] requested) at the end of its run. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpiv_6rqgc/e208c2c7-4c72-4c85-ac78-cd0db9be5621/t0pvytum0:/data', '--log-driver=none', 'aarjunrao/star-fusion:1.0.0', '--chimeric_junction', '/data/STAR.junction', '--output_dir', '/data/fusion-output', '--genome_lib_dir', 'star_with_fusion_100bp_readlen_index', '--CPU', '80.0']' returned non-zero exit status 255. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/fusion.py", line 118, in run_fusion + tool_version=star_fusion_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (255)for command "docker run --rm=true -v /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpiv_6rqgc/e208c2c7-4c72-4c85-ac78-cd0db9be5621/t0pvytum0:/data --log-driver=none aarjunrao/star-fusion:1.0.0 --chimeric_junction /data/STAR.junction --output_dir /data/fusion-output --genome_lib_dir star_with_fusion_100bp_readlen_index --CPU 80.0" + [2020-11-24T01:29:01-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-24T01:29:11-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_fusion' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr to 1 +[2020-11-24T01:29:11-0800] [MainThread] [I] [toil.leader] Issued job 'run_fusion' kind-run_fusion/instance-g9_082sr with job batch system ID: 32 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-24T01:29:12-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9zogi40k/worker_log.txt +[2020-11-24T01:35:11-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-24-2020 01:35:11: Ran phlat on TEST:tumor_rna successfully +[2020-11-24T01:35:12-0800] [MainThread] [I] [toil.leader] Job ended: 'run_phlat' kind-run_phlat/instance-etqwe7dv +[2020-11-24T01:35:12-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-55j01114 with job batch system ID: 33 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-24T01:35:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpw46kediw/worker_log.txt +[2020-11-24T01:35:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpg3iynrfv/worker_log.txt +[2020-11-24T01:35:13-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-55j01114 +[2020-11-24T01:35:13-0800] [MainThread] [I] [toil.leader] Issued job 'merge_phlat_calls' kind-merge_phlat_calls/instance-ss_40opd with job batch system ID: 34 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-24T01:35:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfc79tn8w/worker_log.txt +[2020-11-24T01:35:14-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_phlat_calls' kind-merge_phlat_calls/instance-ss_40opd +[2020-11-24T01:35:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-24-2020 01:35:14: Merging Phlat calls +[2020-11-24T01:35:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-24-2020 01:35:14: Exporting files/for-job/kind-merge_phlat_calls/instance-ss_40opd/file-vndwe_fy/mhci_alleles.list to output location +[2020-11-24T01:35:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-24-2020 01:35:14: Exporting files/for-job/kind-merge_phlat_calls/instance-ss_40opd/file-9hy1brp9/mhcii_alleles.list to output location +[2020-11-24T01:41:55-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_fusion file:/scratch/drkthomp/jobStore kind-run_fusion/instance-g9_082sr. +[2020-11-24T01:41:55-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-24T01:41:55-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-24T01:41:55-0800] [MainThread] [W] [toil.leader] Log from job kind-run_fusion/instance-g9_082sr follows: +=========> + [2020-11-24T01:35:13-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-24T01:35:13-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Error, don't understand arguments: [--CPU 80.0] at /opt/STAR-Fusion-v1.0.0/STAR-Fusion line 167. + [2020-11-24T01:41:46-0800] [MainThread] [W] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-v67rsjos/cleanup/file-d_nkrp7o/stream used 111.60% (59.6 GB [64023433216B] used, 53.4 GB [57369019908B] requested) at the end of its run. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpg3iynrfv/5cf085e6-030a-48be-a946-bde49209bdc5/t5q__6g_m:/data', '--log-driver=none', 'aarjunrao/star-fusion:1.0.0', '--chimeric_junction', '/data/STAR.junction', '--output_dir', '/data/fusion-output', '--genome_lib_dir', 'star_with_fusion_100bp_readlen_index', '--CPU', '80.0']' returned non-zero exit status 255. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/fusion.py", line 118, in run_fusion + tool_version=star_fusion_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (255)for command "docker run --rm=true -v /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpg3iynrfv/5cf085e6-030a-48be-a946-bde49209bdc5/t5q__6g_m:/data --log-driver=none aarjunrao/star-fusion:1.0.0 --chimeric_junction /data/STAR.junction --output_dir /data/fusion-output --genome_lib_dir star_with_fusion_100bp_readlen_index --CPU 80.0" + [2020-11-24T01:41:46-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-24T01:41:55-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_fusion' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr to 0 +[2020-11-24T01:41:55-0800] [MainThread] [W] [toil.leader] Job 'run_fusion' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr is completely failed +[2020-11-24T01:41:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp7jkt14ns/worker_log.txt +[2020-11-24T01:42:55-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_rsem file:/scratch/drkthomp/jobStore kind-run_rsem/instance-mzzok2e3. +[2020-11-24T01:42:55-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-24T01:42:55-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-24T01:42:55-0800] [MainThread] [W] [toil.leader] Log from job kind-run_rsem/instance-mzzok2e3 follows: +=========> + [2020-11-24T01:41:56-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-24T01:41:56-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/expression_profiling/rsem.py", line 88, in run_rsem + dockerhub=univ_options['dockerhub'], tool_version=rsem_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 286, in check_call + retcode = call(*popenargs, **kwargs) + File "/usr/lib64/python3.6/subprocess.py", line 267, in call + with Popen(*popenargs, **kwargs) as p: + File "/usr/lib64/python3.6/subprocess.py", line 709, in __init__ + restore_signals, start_new_session) + File "/usr/lib64/python3.6/subprocess.py", line 1275, in _execute_child + restore_signals, start_new_session, preexec_fn) + TypeError: expected str, bytes or os.PathLike object, not float + [2020-11-24T01:42:54-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-24T01:42:55-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with ID kind-run_rsem/instance-mzzok2e3 to 1 +[2020-11-24T01:42:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with job batch system ID: 35 and cores: 80, disk: 5.0 G, and memory: 2.0 G +[2020-11-24T01:42:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4azb3v8w/worker_log.txt +[2020-11-24T01:43:53-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_rsem file:/scratch/drkthomp/jobStore kind-run_rsem/instance-mzzok2e3. +[2020-11-24T01:43:53-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-24T01:43:53-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-24T01:43:53-0800] [MainThread] [W] [toil.leader] Log from job kind-run_rsem/instance-mzzok2e3 follows: +=========> + [2020-11-24T01:42:56-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-24T01:42:56-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/expression_profiling/rsem.py", line 88, in run_rsem + dockerhub=univ_options['dockerhub'], tool_version=rsem_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 286, in check_call + retcode = call(*popenargs, **kwargs) + File "/usr/lib64/python3.6/subprocess.py", line 267, in call + with Popen(*popenargs, **kwargs) as p: + File "/usr/lib64/python3.6/subprocess.py", line 709, in __init__ + restore_signals, start_new_session) + File "/usr/lib64/python3.6/subprocess.py", line 1275, in _execute_child + restore_signals, start_new_session, preexec_fn) + TypeError: expected str, bytes or os.PathLike object, not float + [2020-11-24T01:43:52-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-24T01:43:53-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with ID kind-run_rsem/instance-mzzok2e3 to 0 +[2020-11-24T01:43:53-0800] [MainThread] [W] [toil.leader] Job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with ID kind-run_rsem/instance-mzzok2e3 is completely failed +[2020-11-24T01:56:45-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-24T02:56:46-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-24T03:56:46-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-24T04:56:47-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-24T05:56:48-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-24T06:56:49-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-24T07:56:49-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-24T08:56:50-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-24T09:56:51-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-24T10:56:51-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-24T11:56:52-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-24T12:56:53-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-24T13:56:53-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-24T14:56:54-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-24T15:56:55-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run diff --git a/errors/2020-11-24-17-00-21.txt b/errors/2020-11-24-17-00-21.txt new file mode 100644 index 00000000..51ca5394 --- /dev/null +++ b/errors/2020-11-24-17-00-21.txt @@ -0,0 +1,408 @@ +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-d9qjg3jl +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-q4vn5u6_ +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-0xdl6glf +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_rsem/instance-mzzok2e3 +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-9uka9hrw +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-7_6wjfvk +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ovr5_kcb +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_fusion/instance-g9_082sr +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-eg7xdcao +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-mppqh5vo +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-lp8p96ga +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-v628y3w0 +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-rd57kqhd +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-bv9m7hf3 +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xz7ir9sh +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r7m8gybn +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-hrzipcl_ +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-k3tx4a2p +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-aux6_f19 +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ewrzdiri +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-06rjxk27 +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-uuibculn +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r1uv5wsv +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-2k5wcr51 +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-yb2yf65u +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-2u48n_5f +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xbtj2suv +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-jkvqxjgf +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ge6vacd7 +[2020-11-24T17:00:23-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-2dtgopje +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-uuibculn with job batch system ID: 0 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-aux6_f19 with job batch system ID: 1 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-d9qjg3jl with job batch system ID: 2 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-eg7xdcao with job batch system ID: 3 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mppqh5vo with job batch system ID: 4 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yb2yf65u with job batch system ID: 5 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-q4vn5u6_ with job batch system ID: 6 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ge6vacd7 with job batch system ID: 7 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-7_6wjfvk with job batch system ID: 8 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2u48n_5f with job batch system ID: 9 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jkvqxjgf with job batch system ID: 10 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bv9m7hf3 with job batch system ID: 11 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r1uv5wsv with job batch system ID: 12 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9uka9hrw with job batch system ID: 13 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ewrzdiri with job batch system ID: 14 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ovr5_kcb with job batch system ID: 15 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xbtj2suv with job batch system ID: 16 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0xdl6glf with job batch system ID: 17 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-v628y3w0 with job batch system ID: 18 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-k3tx4a2p with job batch system ID: 19 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-06rjxk27 with job batch system ID: 20 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-rd57kqhd with job batch system ID: 21 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xz7ir9sh with job batch system ID: 22 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r7m8gybn with job batch system ID: 23 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2dtgopje with job batch system ID: 24 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_fusion' kind-run_fusion/instance-g9_082sr with job batch system ID: 25 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-24T17:00:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with job batch system ID: 26 and cores: 80, disk: 5.0 G, and memory: 2.0 G +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpbcfpe253/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpp11d4y5r/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp2jw13s76/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp1nnhxf0i/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpkc70bd95/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp2pbunwi_/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp9jhd4h4p/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpwbvpd2dq/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpcsa12bih/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpxtv2h0to/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpyiwyl_mk/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpqemi4291/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpdv8rq18r/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmps1cun2wa/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0bqxujvr/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_9nus0tv/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpyz5546g4/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfqb308wj/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp18p8a26f/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpsr_jfesi/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp2yfyrg3z/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp55gyr6_i/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpwd1cjnn2/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpk_jfry9e/worker_log.txt +[2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp25_an2lw/worker_log.txt +[2020-11-24T17:00:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0rjeivhd/worker_log.txt +[2020-11-24T17:00:25-0800] [MainThread] [I] [toil.leader] 26 jobs are running, 1 jobs are issued and waiting to run +[2020-11-24T17:23:38-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_fusion file:/scratch/drkthomp/jobStore kind-run_fusion/instance-g9_082sr. +[2020-11-24T17:23:38-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-24T17:23:38-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-24T17:23:38-0800] [MainThread] [W] [toil.leader] Log from job kind-run_fusion/instance-g9_082sr follows: +=========> + [2020-11-24T17:00:24-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-24T17:00:24-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + CMD: mkdir -p /data/fusion-output + * Running CMD: /opt/STAR-Fusion-v1.0.0/util/STAR-Fusion.predict -J /data/STAR.junction --genome_lib_dir /data/star_with_fusion_100bp_readlen_index --min_junction_reads 1 --min_sum_frags 2 --min_novel_junction_support 3 -O /data/fusion-output/star-fusion + CMD: mkdir -p /data/fusion-output/star-fusion.predict.intermediates_dir + -parsing GTF file: /data/star_with_fusion_100bp_readlen_index/ref_annot.gtf + -building interval tree for fast searching of gene overlaps + -parsing fusion evidence: /data/STAR.junction + -mapping reads to genes + -outputting fusion candidates to file: /data/fusion-output/star-fusion.fusion_candidates.preliminary + * Running CMD: /opt/STAR-Fusion-v1.0.0/util/append_breakpoint_junction_info.pl /data/fusion-output/star-fusion.fusion_candidates.preliminary /data/star_with_fusion_100bp_readlen_index > /data/fusion-output/star-fusion.fusion_candidates.preliminary.wSpliceInfo + [fai_load] build FASTA index. + * Running CMD: /opt/STAR-Fusion-v1.0.0/util/STAR-Fusion.filter --fusion_preds /data/fusion-output/star-fusion.fusion_candidates.preliminary.wSpliceInfo -E 0.001 --tmpdir /tmp --min_junction_reads 1 --min_sum_frags 2 --require_LDAS 1 --max_promiscuity 3 --min_novel_junction_support 3 --min_alt_pct_junction 10 --aggregate_novel_junction_dist 5 --genome_lib_dir /data/star_with_fusion_100bp_readlen_index --out_prefix /data/fusion-output/star-fusion + CMD: mkdir -p /data/fusion-output/star-fusion.filter.intermediates_dir + CMD: /opt/STAR-Fusion-v1.0.0/util/../FusionFilter/blast_and_promiscuity_filter.pl --out_prefix /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion -E 0.001 --max_promiscuity 3 --fusion_preds /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter --genome_lib_dir /data/star_with_fusion_100bp_readlen_index + CMD: /opt/STAR-Fusion-v1.0.0/FusionFilter/util/blast_filter.pl --fusion_preds /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter -E 0.001 --genome_lib_dir /data/star_with_fusion_100bp_readlen_index + CMD: /opt/STAR-Fusion-v1.0.0/FusionFilter/util/promiscuity_filter.pl --fusion_preds /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.post_blast_filter --genome_lib_dir /data/star_with_fusion_100bp_readlen_index --max_promiscuity 3 --min_pct_dom_promiscuity 20 + CMD: cp /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.post_blast_filter.post_promisc_filter /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.final + CMD: /opt/STAR-Fusion-v1.0.0/util/column_exclusions.pl /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter JunctionReads,SpanningFrags > /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.abridged + CMD: /opt/STAR-Fusion-v1.0.0/util/column_exclusions.pl /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.filt_info JunctionReads,SpanningFrags > /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.filt_info.abridged + CMD: /opt/STAR-Fusion-v1.0.0/util/column_exclusions.pl /data/fusion-output/star-fusion.fusion_candidates.final JunctionReads,SpanningFrags > /data/fusion-output/star-fusion.fusion_candidates.final.abridged + CMD: /opt/STAR-Fusion-v1.0.0/util/column_exclusions.pl /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.post_blast_filter.post_promisc_filter JunctionReads,SpanningFrags > /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.post_blast_filter.post_promisc_filter.abridged + + + * Process complete. See output: /data/fusion-output/star-fusion.fusion_candidates.final (or .abridged version) + + + [2020-11-24T17:22:46-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Exporting files/for-job/kind-run_fusion/instance-g9_082sr/file-7j8z7yuq/star-fusion.fusion_candidates.final.abridged to output location + 1.0.1: Pulling from aarjunrao/fusion-inspector + 04c460fac791: Already exists + 0a0916b29f3e: Already exists + b25f4e7a7766: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + 595fd1429200: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + 95b1864271d3: Already exists + 08455334bb58: Already exists + d79e1b284af7: Already exists + 6022965ff030: Already exists + a3ed95caeb02: Already exists + 5397d43b7e0a: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + 08d436ec9dcd: Already exists + 1681d1d31560: Already exists + a056f67462ec: Already exists + 3e6cd62720d9: Already exists + a3ed95caeb02: Already exists + 6ccb234d8efe: Already exists + a3ed95caeb02: Already exists + 4bf02e4e6cf7: Pulling fs layer + 58d3153f09f0: Pulling fs layer + 4bbda8a040c3: Pulling fs layer + 3cfa2f80eba3: Pulling fs layer + 938b83cf2dfb: Pulling fs layer + ae26bd2195d8: Pulling fs layer + 7f9817c0b135: Pulling fs layer + 16f449fc63dd: Pulling fs layer + 488ffd6d1c08: Pulling fs layer + 4ea50944e40a: Pulling fs layer + 938b83cf2dfb: Waiting + 16f449fc63dd: Waiting + 7f9817c0b135: Waiting + 6c51d46ebbc3: Pulling fs layer + 488ffd6d1c08: Waiting + 4ea50944e40a: Waiting + 6c51d46ebbc3: Waiting + 4bbda8a040c3: Verifying Checksum + 4bbda8a040c3: Download complete + 58d3153f09f0: Verifying Checksum + 58d3153f09f0: Download complete + 4bf02e4e6cf7: Verifying Checksum + 4bf02e4e6cf7: Download complete + 938b83cf2dfb: Verifying Checksum + 938b83cf2dfb: Download complete + ae26bd2195d8: Verifying Checksum + ae26bd2195d8: Download complete + 4bf02e4e6cf7: Pull complete + 3cfa2f80eba3: Verifying Checksum + 3cfa2f80eba3: Download complete + 58d3153f09f0: Pull complete + 4bbda8a040c3: Pull complete + 3cfa2f80eba3: Pull complete + 938b83cf2dfb: Pull complete + 488ffd6d1c08: Verifying Checksum + 488ffd6d1c08: Download complete + 7f9817c0b135: Verifying Checksum + 7f9817c0b135: Download complete + ae26bd2195d8: Pull complete + 6c51d46ebbc3: Verifying Checksum + 6c51d46ebbc3: Download complete + 7f9817c0b135: Pull complete + 16f449fc63dd: Verifying Checksum + 16f449fc63dd: Download complete + 4ea50944e40a: Verifying Checksum + 4ea50944e40a: Download complete + 16f449fc63dd: Pull complete + 488ffd6d1c08: Pull complete + 4ea50944e40a: Pull complete + 6c51d46ebbc3: Pull complete + Digest: sha256:a8dfdb8d938005f3dd9cafb54b5c7e5d140a07005c5aae286c15c9c56e49597c + Status: Downloaded newer image for aarjunrao/fusion-inspector:1.0.1 + usage: FusionInspector [-h] --fusions CHIM_SUMMARY_FILES --genome_lib_dir + GENOME_LIB_DIR --left_fq LEFT_FQ_FILENAME --right_fq + RIGHT_FQ_FILENAME --out_prefix OUT_PREFIX + [--align_utils ALIGN_UTILS] + [--min_junction_reads MIN_JUNCTION_READS] + [--min_sum_frags MIN_SUM_FRAGS] + [--min_novel_junction_support MIN_NOVEL_JUNCTION_SUPPORT] + [--require_LDAS REQUIRE_LDAS] + [--max_promiscuity MAX_PROMISCUITY] [-E EVALUE] + [--min_per_id MIN_PER_ID] [--only_fusion_reads] + [--capture_genome_alignments] [--include_Trinity] + [--prep_for_IGV] [--write_intermediate_results] + [--no_cleanup] [--version] [--CPU CPU] [--dirty] + [--aligner_path ALIGNER_PATH] + [--mem_benchmark I_MEM_BENCHMARK] + [--out_dir Output_directory] + FusionInspector: error: argument --CPU: invalid int value: '80.0' + [2020-11-24T17:23:28-0800] [MainThread] [W] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-v67rsjos/cleanup/file-d_nkrp7o/stream used 111.61% (59.6 GB [64028901376B] used, 53.4 GB [57369019908B] requested) at the end of its run. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpyz5546g4/bc43a4b5-98d1-4e5d-b16a-21548bd92604/t4azbndxa:/data', '--log-driver=none', 'aarjunrao/fusion-inspector:1.0.1', '--fusions', '/data/fusion-output/star-fusion.fusion_candidates.final.abridged', '--genome_lib', 'star_with_fusion_100bp_readlen_index', '--left_fq', '/data/rna_1.fq.gz', '--right_fq', '/data/rna_2.fq.gz', '--out_dir', '/data/FusionInspector', '--out_prefix', 'FusionInspector', '--CPU', '80.0', '--include_Trinity']' returned non-zero exit status 2. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/fusion.py", line 154, in run_fusion + tool_version=fusion_inspector_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (2)for command "docker run --rm=true -v /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpyz5546g4/bc43a4b5-98d1-4e5d-b16a-21548bd92604/t4azbndxa:/data --log-driver=none aarjunrao/fusion-inspector:1.0.1 --fusions /data/fusion-output/star-fusion.fusion_candidates.final.abridged --genome_lib star_with_fusion_100bp_readlen_index --left_fq /data/rna_1.fq.gz --right_fq /data/rna_2.fq.gz --out_dir /data/FusionInspector --out_prefix FusionInspector --CPU 80.0 --include_Trinity" + [2020-11-24T17:23:28-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-24T17:23:38-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_fusion' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr to 1 +[2020-11-24T17:23:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_fusion' kind-run_fusion/instance-g9_082sr with job batch system ID: 27 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-24T17:23:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpy4anvbbo/worker_log.txt +[2020-11-24T17:24:34-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_rsem file:/scratch/drkthomp/jobStore kind-run_rsem/instance-mzzok2e3. +[2020-11-24T17:24:34-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-24T17:24:34-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-24T17:24:34-0800] [MainThread] [W] [toil.leader] Log from job kind-run_rsem/instance-mzzok2e3 follows: +=========> + [2020-11-24T17:23:39-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-24T17:23:39-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/expression_profiling/rsem.py", line 88, in run_rsem + dockerhub=univ_options['dockerhub'], tool_version=rsem_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 286, in check_call + retcode = call(*popenargs, **kwargs) + File "/usr/lib64/python3.6/subprocess.py", line 267, in call + with Popen(*popenargs, **kwargs) as p: + File "/usr/lib64/python3.6/subprocess.py", line 709, in __init__ + restore_signals, start_new_session) + File "/usr/lib64/python3.6/subprocess.py", line 1275, in _execute_child + restore_signals, start_new_session, preexec_fn) + TypeError: expected str, bytes or os.PathLike object, not int + [2020-11-24T17:24:33-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-24T17:24:34-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with ID kind-run_rsem/instance-mzzok2e3 to 1 +[2020-11-24T17:24:34-0800] [MainThread] [I] [toil.leader] Issued job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with job batch system ID: 28 and cores: 80, disk: 5.0 G, and memory: 2.0 G +[2020-11-24T17:24:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpemf4jdqr/worker_log.txt +[2020-11-24T17:34:35-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_fusion file:/scratch/drkthomp/jobStore kind-run_fusion/instance-g9_082sr. +[2020-11-24T17:34:35-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-24T17:34:35-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-24T17:34:35-0800] [MainThread] [W] [toil.leader] Log from job kind-run_fusion/instance-g9_082sr follows: +=========> + [2020-11-24T17:24:35-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-24T17:24:35-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + CMD: mkdir -p /data/fusion-output + * Running CMD: /opt/STAR-Fusion-v1.0.0/util/STAR-Fusion.predict -J /data/STAR.junction --genome_lib_dir /data/star_with_fusion_100bp_readlen_index --min_junction_reads 1 --min_sum_frags 2 --min_novel_junction_support 3 -O /data/fusion-output/star-fusion + CMD: mkdir -p /data/fusion-output/star-fusion.predict.intermediates_dir + -parsing GTF file: /data/star_with_fusion_100bp_readlen_index/ref_annot.gtf + -building interval tree for fast searching of gene overlaps + -parsing fusion evidence: /data/STAR.junction + -mapping reads to genes + -outputting fusion candidates to file: /data/fusion-output/star-fusion.fusion_candidates.preliminary + * Running CMD: /opt/STAR-Fusion-v1.0.0/util/append_breakpoint_junction_info.pl /data/fusion-output/star-fusion.fusion_candidates.preliminary /data/star_with_fusion_100bp_readlen_index > /data/fusion-output/star-fusion.fusion_candidates.preliminary.wSpliceInfo + [fai_load] build FASTA index. + * Running CMD: /opt/STAR-Fusion-v1.0.0/util/STAR-Fusion.filter --fusion_preds /data/fusion-output/star-fusion.fusion_candidates.preliminary.wSpliceInfo -E 0.001 --tmpdir /tmp --min_junction_reads 1 --min_sum_frags 2 --require_LDAS 1 --max_promiscuity 3 --min_novel_junction_support 3 --min_alt_pct_junction 10 --aggregate_novel_junction_dist 5 --genome_lib_dir /data/star_with_fusion_100bp_readlen_index --out_prefix /data/fusion-output/star-fusion + CMD: mkdir -p /data/fusion-output/star-fusion.filter.intermediates_dir + CMD: /opt/STAR-Fusion-v1.0.0/util/../FusionFilter/blast_and_promiscuity_filter.pl --out_prefix /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion -E 0.001 --max_promiscuity 3 --fusion_preds /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter --genome_lib_dir /data/star_with_fusion_100bp_readlen_index + CMD: /opt/STAR-Fusion-v1.0.0/FusionFilter/util/blast_filter.pl --fusion_preds /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter -E 0.001 --genome_lib_dir /data/star_with_fusion_100bp_readlen_index + CMD: /opt/STAR-Fusion-v1.0.0/FusionFilter/util/promiscuity_filter.pl --fusion_preds /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.post_blast_filter --genome_lib_dir /data/star_with_fusion_100bp_readlen_index --max_promiscuity 3 --min_pct_dom_promiscuity 20 + CMD: cp /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.post_blast_filter.post_promisc_filter /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.final + CMD: /opt/STAR-Fusion-v1.0.0/util/column_exclusions.pl /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter JunctionReads,SpanningFrags > /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.abridged + CMD: /opt/STAR-Fusion-v1.0.0/util/column_exclusions.pl /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.filt_info JunctionReads,SpanningFrags > /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.filt_info.abridged + CMD: /opt/STAR-Fusion-v1.0.0/util/column_exclusions.pl /data/fusion-output/star-fusion.fusion_candidates.final JunctionReads,SpanningFrags > /data/fusion-output/star-fusion.fusion_candidates.final.abridged + CMD: /opt/STAR-Fusion-v1.0.0/util/column_exclusions.pl /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.post_blast_filter.post_promisc_filter JunctionReads,SpanningFrags > /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.post_blast_filter.post_promisc_filter.abridged + + + * Process complete. See output: /data/fusion-output/star-fusion.fusion_candidates.final (or .abridged version) + + + [2020-11-24T17:34:24-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Exporting files/for-job/kind-run_fusion/instance-g9_082sr/file-00gboftw/star-fusion.fusion_candidates.final.abridged to output location + usage: FusionInspector [-h] --fusions CHIM_SUMMARY_FILES --genome_lib_dir + GENOME_LIB_DIR --left_fq LEFT_FQ_FILENAME --right_fq + RIGHT_FQ_FILENAME --out_prefix OUT_PREFIX + [--align_utils ALIGN_UTILS] + [--min_junction_reads MIN_JUNCTION_READS] + [--min_sum_frags MIN_SUM_FRAGS] + [--min_novel_junction_support MIN_NOVEL_JUNCTION_SUPPORT] + [--require_LDAS REQUIRE_LDAS] + [--max_promiscuity MAX_PROMISCUITY] [-E EVALUE] + [--min_per_id MIN_PER_ID] [--only_fusion_reads] + [--capture_genome_alignments] [--include_Trinity] + [--prep_for_IGV] [--write_intermediate_results] + [--no_cleanup] [--version] [--CPU CPU] [--dirty] + [--aligner_path ALIGNER_PATH] + [--mem_benchmark I_MEM_BENCHMARK] + [--out_dir Output_directory] + FusionInspector: error: argument --CPU: invalid int value: '80.0' + [2020-11-24T17:34:26-0800] [MainThread] [W] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-v67rsjos/cleanup/file-d_nkrp7o/stream used 111.61% (59.6 GB [64028901376B] used, 53.4 GB [57369019908B] requested) at the end of its run. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpemf4jdqr/38ee5094-cf45-4cb8-9184-63af240a9b8a/t8qnl3oju:/data', '--log-driver=none', 'aarjunrao/fusion-inspector:1.0.1', '--fusions', '/data/fusion-output/star-fusion.fusion_candidates.final.abridged', '--genome_lib', 'star_with_fusion_100bp_readlen_index', '--left_fq', '/data/rna_1.fq.gz', '--right_fq', '/data/rna_2.fq.gz', '--out_dir', '/data/FusionInspector', '--out_prefix', 'FusionInspector', '--CPU', '80.0', '--include_Trinity']' returned non-zero exit status 2. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/fusion.py", line 154, in run_fusion + tool_version=fusion_inspector_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (2)for command "docker run --rm=true -v /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpemf4jdqr/38ee5094-cf45-4cb8-9184-63af240a9b8a/t8qnl3oju:/data --log-driver=none aarjunrao/fusion-inspector:1.0.1 --fusions /data/fusion-output/star-fusion.fusion_candidates.final.abridged --genome_lib star_with_fusion_100bp_readlen_index --left_fq /data/rna_1.fq.gz --right_fq /data/rna_2.fq.gz --out_dir /data/FusionInspector --out_prefix FusionInspector --CPU 80.0 --include_Trinity" + [2020-11-24T17:34:26-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-24T17:34:35-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_fusion' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr to 0 +[2020-11-24T17:34:35-0800] [MainThread] [W] [toil.leader] Job 'run_fusion' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr is completely failed +[2020-11-24T17:34:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0nh79xal/worker_log.txt +[2020-11-24T17:35:32-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_rsem file:/scratch/drkthomp/jobStore kind-run_rsem/instance-mzzok2e3. +[2020-11-24T17:35:32-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-24T17:35:32-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-24T17:35:32-0800] [MainThread] [W] [toil.leader] Log from job kind-run_rsem/instance-mzzok2e3 follows: +=========> + [2020-11-24T17:34:36-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-24T17:34:36-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/expression_profiling/rsem.py", line 88, in run_rsem + dockerhub=univ_options['dockerhub'], tool_version=rsem_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 286, in check_call + retcode = call(*popenargs, **kwargs) + File "/usr/lib64/python3.6/subprocess.py", line 267, in call + with Popen(*popenargs, **kwargs) as p: + File "/usr/lib64/python3.6/subprocess.py", line 709, in __init__ + restore_signals, start_new_session) + File "/usr/lib64/python3.6/subprocess.py", line 1275, in _execute_child + restore_signals, start_new_session, preexec_fn) + TypeError: expected str, bytes or os.PathLike object, not int + [2020-11-24T17:35:31-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-24T17:35:32-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with ID kind-run_rsem/instance-mzzok2e3 to 0 +[2020-11-24T17:35:32-0800] [MainThread] [W] [toil.leader] Job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with ID kind-run_rsem/instance-mzzok2e3 is completely failed +[2020-11-24T18:00:26-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-24T19:00:27-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-24T20:00:28-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-24T21:00:28-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-24T22:00:29-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-24T23:00:30-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T00:00:30-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T01:00:31-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T02:00:32-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T03:00:33-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T04:00:34-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T05:00:34-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T06:00:35-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T07:00:36-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T08:00:37-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T09:00:37-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T10:00:38-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T11:00:38-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T12:00:39-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T13:00:40-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T14:00:40-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run diff --git a/errors/2020-11-25-14-55-40.txt b/errors/2020-11-25-14-55-40.txt new file mode 100644 index 00000000..07f3c765 --- /dev/null +++ b/errors/2020-11-25-14-55-40.txt @@ -0,0 +1,307 @@ +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_rsem/instance-mzzok2e3 +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-9uka9hrw +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ewrzdiri +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-bv9m7hf3 +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ge6vacd7 +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-mppqh5vo +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-0xdl6glf +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xz7ir9sh +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_fusion/instance-g9_082sr +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-aux6_f19 +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-k3tx4a2p +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ovr5_kcb +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-2dtgopje +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-yb2yf65u +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-eg7xdcao +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r1uv5wsv +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-uuibculn +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-rd57kqhd +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-2u48n_5f +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-7_6wjfvk +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-q4vn5u6_ +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r7m8gybn +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-06rjxk27 +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-d9qjg3jl +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-jkvqxjgf +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xbtj2suv +[2020-11-25T14:55:42-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-v628y3w0 +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-uuibculn with job batch system ID: 0 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-aux6_f19 with job batch system ID: 1 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-d9qjg3jl with job batch system ID: 2 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-eg7xdcao with job batch system ID: 3 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mppqh5vo with job batch system ID: 4 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yb2yf65u with job batch system ID: 5 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-q4vn5u6_ with job batch system ID: 6 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ge6vacd7 with job batch system ID: 7 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-7_6wjfvk with job batch system ID: 8 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2u48n_5f with job batch system ID: 9 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jkvqxjgf with job batch system ID: 10 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bv9m7hf3 with job batch system ID: 11 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r1uv5wsv with job batch system ID: 12 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9uka9hrw with job batch system ID: 13 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ewrzdiri with job batch system ID: 14 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ovr5_kcb with job batch system ID: 15 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xbtj2suv with job batch system ID: 16 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0xdl6glf with job batch system ID: 17 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-v628y3w0 with job batch system ID: 18 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-k3tx4a2p with job batch system ID: 19 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-06rjxk27 with job batch system ID: 20 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-rd57kqhd with job batch system ID: 21 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xz7ir9sh with job batch system ID: 22 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r7m8gybn with job batch system ID: 23 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2dtgopje with job batch system ID: 24 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_fusion' kind-run_fusion/instance-g9_082sr with job batch system ID: 25 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-25T14:55:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with job batch system ID: 26 and cores: 80, disk: 5.0 G, and memory: 2.0 G +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpn6lzg_s6/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpaz5fft1v/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpq9nz2aiv/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpzv0g2max/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmppxhjj97t/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpr41r79ez/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_erovlir/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpyz3ien50/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmps_9wbaqv/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpnd2pxcsa/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpf9vc394k/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpgss8lmn2/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_oz0gi52/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfei4w50p/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpqwvzvt25/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp08i9uthr/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmplcuu0cw0/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpqp50zfi6/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0yv11o8n/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpwo9relq_/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpstz55naq/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmps8a2mh1z/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpio024aw2/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfdhde7hw/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpuuk6v2f8/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpdl7i15rz/worker_log.txt +[2020-11-25T14:55:44-0800] [MainThread] [I] [toil.leader] 26 jobs are running, 1 jobs are issued and waiting to run +[2020-11-25T15:17:44-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_fusion file:/scratch/drkthomp/jobStore kind-run_fusion/instance-g9_082sr. +[2020-11-25T15:17:44-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-25T15:17:44-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-25T15:17:44-0800] [MainThread] [W] [toil.leader] Log from job kind-run_fusion/instance-g9_082sr follows: +=========> + [2020-11-25T14:55:44-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-25T14:55:44-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + CMD: mkdir -p /data/fusion-output + * Running CMD: /opt/STAR-Fusion-v1.0.0/util/STAR-Fusion.predict -J /data/STAR.junction --genome_lib_dir /data/star_with_fusion_100bp_readlen_index --min_junction_reads 1 --min_sum_frags 2 --min_novel_junction_support 3 -O /data/fusion-output/star-fusion + CMD: mkdir -p /data/fusion-output/star-fusion.predict.intermediates_dir + -parsing GTF file: /data/star_with_fusion_100bp_readlen_index/ref_annot.gtf + -building interval tree for fast searching of gene overlaps + -parsing fusion evidence: /data/STAR.junction + -mapping reads to genes + -outputting fusion candidates to file: /data/fusion-output/star-fusion.fusion_candidates.preliminary + * Running CMD: /opt/STAR-Fusion-v1.0.0/util/append_breakpoint_junction_info.pl /data/fusion-output/star-fusion.fusion_candidates.preliminary /data/star_with_fusion_100bp_readlen_index > /data/fusion-output/star-fusion.fusion_candidates.preliminary.wSpliceInfo + [fai_load] build FASTA index. + * Running CMD: /opt/STAR-Fusion-v1.0.0/util/STAR-Fusion.filter --fusion_preds /data/fusion-output/star-fusion.fusion_candidates.preliminary.wSpliceInfo -E 0.001 --tmpdir /tmp --min_junction_reads 1 --min_sum_frags 2 --require_LDAS 1 --max_promiscuity 3 --min_novel_junction_support 3 --min_alt_pct_junction 10 --aggregate_novel_junction_dist 5 --genome_lib_dir /data/star_with_fusion_100bp_readlen_index --out_prefix /data/fusion-output/star-fusion + CMD: mkdir -p /data/fusion-output/star-fusion.filter.intermediates_dir + CMD: /opt/STAR-Fusion-v1.0.0/util/../FusionFilter/blast_and_promiscuity_filter.pl --out_prefix /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion -E 0.001 --max_promiscuity 3 --fusion_preds /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter --genome_lib_dir /data/star_with_fusion_100bp_readlen_index + CMD: /opt/STAR-Fusion-v1.0.0/FusionFilter/util/blast_filter.pl --fusion_preds /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter -E 0.001 --genome_lib_dir /data/star_with_fusion_100bp_readlen_index + CMD: /opt/STAR-Fusion-v1.0.0/FusionFilter/util/promiscuity_filter.pl --fusion_preds /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.post_blast_filter --genome_lib_dir /data/star_with_fusion_100bp_readlen_index --max_promiscuity 3 --min_pct_dom_promiscuity 20 + CMD: cp /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.post_blast_filter.post_promisc_filter /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.final + CMD: /opt/STAR-Fusion-v1.0.0/util/column_exclusions.pl /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter JunctionReads,SpanningFrags > /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.abridged + CMD: /opt/STAR-Fusion-v1.0.0/util/column_exclusions.pl /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.filt_info JunctionReads,SpanningFrags > /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.filt_info.abridged + CMD: /opt/STAR-Fusion-v1.0.0/util/column_exclusions.pl /data/fusion-output/star-fusion.fusion_candidates.final JunctionReads,SpanningFrags > /data/fusion-output/star-fusion.fusion_candidates.final.abridged + CMD: /opt/STAR-Fusion-v1.0.0/util/column_exclusions.pl /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.post_blast_filter.post_promisc_filter JunctionReads,SpanningFrags > /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.post_blast_filter.post_promisc_filter.abridged + + + * Process complete. See output: /data/fusion-output/star-fusion.fusion_candidates.final (or .abridged version) + + + [2020-11-25T15:17:32-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Exporting files/for-job/kind-run_fusion/instance-g9_082sr/file-4vsnimwt/star-fusion.fusion_candidates.final.abridged to output location + usage: FusionInspector [-h] --fusions CHIM_SUMMARY_FILES --genome_lib_dir + GENOME_LIB_DIR --left_fq LEFT_FQ_FILENAME --right_fq + RIGHT_FQ_FILENAME --out_prefix OUT_PREFIX + [--align_utils ALIGN_UTILS] + [--min_junction_reads MIN_JUNCTION_READS] + [--min_sum_frags MIN_SUM_FRAGS] + [--min_novel_junction_support MIN_NOVEL_JUNCTION_SUPPORT] + [--require_LDAS REQUIRE_LDAS] + [--max_promiscuity MAX_PROMISCUITY] [-E EVALUE] + [--min_per_id MIN_PER_ID] [--only_fusion_reads] + [--capture_genome_alignments] [--include_Trinity] + [--prep_for_IGV] [--write_intermediate_results] + [--no_cleanup] [--version] [--CPU CPU] [--dirty] + [--aligner_path ALIGNER_PATH] + [--mem_benchmark I_MEM_BENCHMARK] + [--out_dir Output_directory] + FusionInspector: error: argument --CPU: invalid int value: '80.0' + [2020-11-25T15:17:35-0800] [MainThread] [W] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-v67rsjos/cleanup/file-d_nkrp7o/stream used 111.61% (59.6 GB [64028901376B] used, 53.4 GB [57369019908B] requested) at the end of its run. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0yv11o8n/97a2ce2d-acb4-407d-bc6f-d53c2ffc9032/tkgtpjbp1:/data', '--log-driver=none', 'aarjunrao/fusion-inspector:1.0.1', '--fusions', '/data/fusion-output/star-fusion.fusion_candidates.final.abridged', '--genome_lib', 'star_with_fusion_100bp_readlen_index', '--left_fq', '/data/rna_1.fq.gz', '--right_fq', '/data/rna_2.fq.gz', '--out_dir', '/data/FusionInspector', '--out_prefix', 'FusionInspector', '--CPU', '80.0', '--include_Trinity']' returned non-zero exit status 2. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/fusion.py", line 154, in run_fusion + tool_version=fusion_inspector_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (2)for command "docker run --rm=true -v /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0yv11o8n/97a2ce2d-acb4-407d-bc6f-d53c2ffc9032/tkgtpjbp1:/data --log-driver=none aarjunrao/fusion-inspector:1.0.1 --fusions /data/fusion-output/star-fusion.fusion_candidates.final.abridged --genome_lib star_with_fusion_100bp_readlen_index --left_fq /data/rna_1.fq.gz --right_fq /data/rna_2.fq.gz --out_dir /data/FusionInspector --out_prefix FusionInspector --CPU 80.0 --include_Trinity" + [2020-11-25T15:17:35-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-25T15:17:44-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_fusion' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr to 1 +[2020-11-25T15:17:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_fusion' kind-run_fusion/instance-g9_082sr with job batch system ID: 27 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-25T15:17:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpdzqaut4w/worker_log.txt +[2020-11-25T15:18:46-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_rsem file:/scratch/drkthomp/jobStore kind-run_rsem/instance-mzzok2e3. +[2020-11-25T15:18:46-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-25T15:18:46-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-25T15:18:46-0800] [MainThread] [W] [toil.leader] Log from job kind-run_rsem/instance-mzzok2e3 follows: +=========> + [2020-11-25T15:17:44-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-25T15:17:44-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/expression_profiling/rsem.py", line 88, in run_rsem + dockerhub=univ_options['dockerhub'], tool_version=rsem_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 286, in check_call + retcode = call(*popenargs, **kwargs) + File "/usr/lib64/python3.6/subprocess.py", line 267, in call + with Popen(*popenargs, **kwargs) as p: + File "/usr/lib64/python3.6/subprocess.py", line 709, in __init__ + restore_signals, start_new_session) + File "/usr/lib64/python3.6/subprocess.py", line 1275, in _execute_child + restore_signals, start_new_session, preexec_fn) + TypeError: expected str, bytes or os.PathLike object, not int + [2020-11-25T15:18:45-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-25T15:18:46-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with ID kind-run_rsem/instance-mzzok2e3 to 1 +[2020-11-25T15:18:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with job batch system ID: 28 and cores: 80, disk: 5.0 G, and memory: 2.0 G +[2020-11-25T15:18:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpx6h2rig3/worker_log.txt +[2020-11-25T15:29:01-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_fusion file:/scratch/drkthomp/jobStore kind-run_fusion/instance-g9_082sr. +[2020-11-25T15:29:01-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-25T15:29:01-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-25T15:29:01-0800] [MainThread] [W] [toil.leader] Log from job kind-run_fusion/instance-g9_082sr follows: +=========> + [2020-11-25T15:18:47-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-25T15:18:47-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + CMD: mkdir -p /data/fusion-output + * Running CMD: /opt/STAR-Fusion-v1.0.0/util/STAR-Fusion.predict -J /data/STAR.junction --genome_lib_dir /data/star_with_fusion_100bp_readlen_index --min_junction_reads 1 --min_sum_frags 2 --min_novel_junction_support 3 -O /data/fusion-output/star-fusion + CMD: mkdir -p /data/fusion-output/star-fusion.predict.intermediates_dir + -parsing GTF file: /data/star_with_fusion_100bp_readlen_index/ref_annot.gtf + -building interval tree for fast searching of gene overlaps + -parsing fusion evidence: /data/STAR.junction + -mapping reads to genes + -outputting fusion candidates to file: /data/fusion-output/star-fusion.fusion_candidates.preliminary + * Running CMD: /opt/STAR-Fusion-v1.0.0/util/append_breakpoint_junction_info.pl /data/fusion-output/star-fusion.fusion_candidates.preliminary /data/star_with_fusion_100bp_readlen_index > /data/fusion-output/star-fusion.fusion_candidates.preliminary.wSpliceInfo + [fai_load] build FASTA index. + * Running CMD: /opt/STAR-Fusion-v1.0.0/util/STAR-Fusion.filter --fusion_preds /data/fusion-output/star-fusion.fusion_candidates.preliminary.wSpliceInfo -E 0.001 --tmpdir /tmp --min_junction_reads 1 --min_sum_frags 2 --require_LDAS 1 --max_promiscuity 3 --min_novel_junction_support 3 --min_alt_pct_junction 10 --aggregate_novel_junction_dist 5 --genome_lib_dir /data/star_with_fusion_100bp_readlen_index --out_prefix /data/fusion-output/star-fusion + CMD: mkdir -p /data/fusion-output/star-fusion.filter.intermediates_dir + CMD: /opt/STAR-Fusion-v1.0.0/util/../FusionFilter/blast_and_promiscuity_filter.pl --out_prefix /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion -E 0.001 --max_promiscuity 3 --fusion_preds /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter --genome_lib_dir /data/star_with_fusion_100bp_readlen_index + CMD: /opt/STAR-Fusion-v1.0.0/FusionFilter/util/blast_filter.pl --fusion_preds /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter -E 0.001 --genome_lib_dir /data/star_with_fusion_100bp_readlen_index + CMD: /opt/STAR-Fusion-v1.0.0/FusionFilter/util/promiscuity_filter.pl --fusion_preds /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.post_blast_filter --genome_lib_dir /data/star_with_fusion_100bp_readlen_index --max_promiscuity 3 --min_pct_dom_promiscuity 20 + CMD: cp /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.post_blast_filter.post_promisc_filter /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.final + CMD: /opt/STAR-Fusion-v1.0.0/util/column_exclusions.pl /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter JunctionReads,SpanningFrags > /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.abridged + CMD: /opt/STAR-Fusion-v1.0.0/util/column_exclusions.pl /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.filt_info JunctionReads,SpanningFrags > /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.filt_info.abridged + CMD: /opt/STAR-Fusion-v1.0.0/util/column_exclusions.pl /data/fusion-output/star-fusion.fusion_candidates.final JunctionReads,SpanningFrags > /data/fusion-output/star-fusion.fusion_candidates.final.abridged + CMD: /opt/STAR-Fusion-v1.0.0/util/column_exclusions.pl /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.post_blast_filter.post_promisc_filter JunctionReads,SpanningFrags > /data/fusion-output/star-fusion.filter.intermediates_dir/star-fusion.pre_blast_filter.post_blast_filter.post_promisc_filter.abridged + + + * Process complete. See output: /data/fusion-output/star-fusion.fusion_candidates.final (or .abridged version) + + + [2020-11-25T15:28:50-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Exporting files/for-job/kind-run_fusion/instance-g9_082sr/file-mkeepob_/star-fusion.fusion_candidates.final.abridged to output location + usage: FusionInspector [-h] --fusions CHIM_SUMMARY_FILES --genome_lib_dir + GENOME_LIB_DIR --left_fq LEFT_FQ_FILENAME --right_fq + RIGHT_FQ_FILENAME --out_prefix OUT_PREFIX + [--align_utils ALIGN_UTILS] + [--min_junction_reads MIN_JUNCTION_READS] + [--min_sum_frags MIN_SUM_FRAGS] + [--min_novel_junction_support MIN_NOVEL_JUNCTION_SUPPORT] + [--require_LDAS REQUIRE_LDAS] + [--max_promiscuity MAX_PROMISCUITY] [-E EVALUE] + [--min_per_id MIN_PER_ID] [--only_fusion_reads] + [--capture_genome_alignments] [--include_Trinity] + [--prep_for_IGV] [--write_intermediate_results] + [--no_cleanup] [--version] [--CPU CPU] [--dirty] + [--aligner_path ALIGNER_PATH] + [--mem_benchmark I_MEM_BENCHMARK] + [--out_dir Output_directory] + FusionInspector: error: argument --CPU: invalid int value: '80.0' + [2020-11-25T15:28:52-0800] [MainThread] [W] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-v67rsjos/cleanup/file-d_nkrp7o/stream used 111.61% (59.6 GB [64028901376B] used, 53.4 GB [57369019908B] requested) at the end of its run. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpx6h2rig3/1e3dd742-2688-44a0-912c-688bf411a65f/tiekp8xcd:/data', '--log-driver=none', 'aarjunrao/fusion-inspector:1.0.1', '--fusions', '/data/fusion-output/star-fusion.fusion_candidates.final.abridged', '--genome_lib', 'star_with_fusion_100bp_readlen_index', '--left_fq', '/data/rna_1.fq.gz', '--right_fq', '/data/rna_2.fq.gz', '--out_dir', '/data/FusionInspector', '--out_prefix', 'FusionInspector', '--CPU', '80.0', '--include_Trinity']' returned non-zero exit status 2. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/fusion.py", line 154, in run_fusion + tool_version=fusion_inspector_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 139, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (2)for command "docker run --rm=true -v /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpx6h2rig3/1e3dd742-2688-44a0-912c-688bf411a65f/tiekp8xcd:/data --log-driver=none aarjunrao/fusion-inspector:1.0.1 --fusions /data/fusion-output/star-fusion.fusion_candidates.final.abridged --genome_lib star_with_fusion_100bp_readlen_index --left_fq /data/rna_1.fq.gz --right_fq /data/rna_2.fq.gz --out_dir /data/FusionInspector --out_prefix FusionInspector --CPU 80.0 --include_Trinity" + [2020-11-25T15:28:52-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-25T15:29:01-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_fusion' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr to 0 +[2020-11-25T15:29:01-0800] [MainThread] [W] [toil.leader] Job 'run_fusion' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr is completely failed +[2020-11-25T15:29:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpn3axwubh/worker_log.txt +[2020-11-25T15:29:58-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_rsem file:/scratch/drkthomp/jobStore kind-run_rsem/instance-mzzok2e3. +[2020-11-25T15:29:58-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-25T15:29:58-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-25T15:29:58-0800] [MainThread] [W] [toil.leader] Log from job kind-run_rsem/instance-mzzok2e3 follows: +=========> + [2020-11-25T15:29:02-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-25T15:29:02-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/expression_profiling/rsem.py", line 88, in run_rsem + dockerhub=univ_options['dockerhub'], tool_version=rsem_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 286, in check_call + retcode = call(*popenargs, **kwargs) + File "/usr/lib64/python3.6/subprocess.py", line 267, in call + with Popen(*popenargs, **kwargs) as p: + File "/usr/lib64/python3.6/subprocess.py", line 709, in __init__ + restore_signals, start_new_session) + File "/usr/lib64/python3.6/subprocess.py", line 1275, in _execute_child + restore_signals, start_new_session, preexec_fn) + TypeError: expected str, bytes or os.PathLike object, not int + [2020-11-25T15:29:57-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-25T15:29:58-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with ID kind-run_rsem/instance-mzzok2e3 to 0 +[2020-11-25T15:29:58-0800] [MainThread] [W] [toil.leader] Job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with ID kind-run_rsem/instance-mzzok2e3 is completely failed diff --git a/errors/2020-11-25-16-10-09.txt b/errors/2020-11-25-16-10-09.txt new file mode 100644 index 00000000..2a45c9f9 --- /dev/null +++ b/errors/2020-11-25-16-10-09.txt @@ -0,0 +1,720 @@ +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-eg7xdcao +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r1uv5wsv +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-bv9m7hf3 +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-d9qjg3jl +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-uuibculn +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-aux6_f19 +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-k3tx4a2p +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ewrzdiri +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-q4vn5u6_ +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-2u48n_5f +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-7_6wjfvk +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-rd57kqhd +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ovr5_kcb +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-2dtgopje +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-v628y3w0 +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xz7ir9sh +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r7m8gybn +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-yb2yf65u +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-06rjxk27 +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-9uka9hrw +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_rsem/instance-mzzok2e3 +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ge6vacd7 +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-jkvqxjgf +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-mppqh5vo +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xbtj2suv +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_fusion/instance-g9_082sr +[2020-11-25T16:10:12-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-0xdl6glf +[2020-11-25T16:10:12-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-uuibculn with job batch system ID: 0 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-aux6_f19 with job batch system ID: 1 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-d9qjg3jl with job batch system ID: 2 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-eg7xdcao with job batch system ID: 3 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mppqh5vo with job batch system ID: 4 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yb2yf65u with job batch system ID: 5 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-q4vn5u6_ with job batch system ID: 6 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ge6vacd7 with job batch system ID: 7 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-7_6wjfvk with job batch system ID: 8 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2u48n_5f with job batch system ID: 9 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jkvqxjgf with job batch system ID: 10 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bv9m7hf3 with job batch system ID: 11 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r1uv5wsv with job batch system ID: 12 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9uka9hrw with job batch system ID: 13 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ewrzdiri with job batch system ID: 14 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ovr5_kcb with job batch system ID: 15 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xbtj2suv with job batch system ID: 16 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0xdl6glf with job batch system ID: 17 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-v628y3w0 with job batch system ID: 18 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-k3tx4a2p with job batch system ID: 19 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-06rjxk27 with job batch system ID: 20 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-rd57kqhd with job batch system ID: 21 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xz7ir9sh with job batch system ID: 22 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r7m8gybn with job batch system ID: 23 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2dtgopje with job batch system ID: 24 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_fusion' kind-run_fusion/instance-g9_082sr with job batch system ID: 25 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-25T16:10:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_rsem' kind-run_rsem/instance-mzzok2e3 with job batch system ID: 26 and cores: 80, disk: 5.0 G, and memory: 2.0 G +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpcwt95pgg/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpuxv8ajdk/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmponi0h7ui/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0bo6_nr0/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp5t99t0go/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp708badqu/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpa_x61_2p/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpkrhjk_xu/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpgtwv6bw2/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpow110ksj/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpk3lybk94/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmps09gi1fx/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpwl_chi3i/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp08behm86/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpndacuvu0/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpc02ewl_i/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp6c44htg_/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpof9_h1x6/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjt84bnj8/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpf93gkydd/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp__fvc1l8/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpntmpkj_x/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmprf_4600_/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpts4nqkne/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjtc_q3fl/worker_log.txt +[2020-11-25T16:10:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpc9k72_a2/worker_log.txt +[2020-11-25T16:10:15-0800] [MainThread] [I] [toil.leader] 26 jobs are running, 1 jobs are issued and waiting to run +[2020-11-25T16:46:34-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_fusion file:/scratch/drkthomp/jobStore kind-run_fusion/instance-g9_082sr. +[2020-11-25T16:46:34-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_fusion' kind-run_fusion/instance-g9_082sr +[2020-11-25T16:46:34-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'reformat_star_fusion_output' kind-run_fusion/instance-g9_082sr +[2020-11-25T16:46:34-0800] [MainThread] [W] [toil.leader] Log from job kind-run_fusion/instance-g9_082sr follows: +=========> + + 2020-11-26 00:45:36,048 - FusionInspector - INFO - DependencyTree.func_complete_command: Products are made + 2020-11-26 00:45:36,048 - FusionInspector - INFO - DependencyTree.func_complete_command: /data/FusionInspector/FusionInspector.spanning_reads.bam.bai 208.0 B + 2020-11-26 00:45:36,048 - FusionInspector - INFO - Pipeline.func_run_commands: Updated dependencies. True + 2020-11-26 00:45:36,048 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:45:36,048 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.spanning_reads.bam.bai was made. + 2020-11-26 00:45:36,048 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:45:36,049 - FusionInspector - INFO - Pipeline.func_run_commands: Time:: 0.0 + 2020-11-26 00:45:36,049 - FusionInspector - INFO - + + Pipeline.func_run_commands: Starting /opt/FusionInspector-v1.0.1/util/consolidate_bams_and_uniq_reads.pl /data/FusionInspector/fi_workdir/FusionInspector.fa /data/FusionInspector/fi_workdir/FusionInspector.star2.cSorted.rmdup.bam /data/FusionInspector/FusionInspector.consolidated + 2020-11-26 00:45:36,049 - FusionInspector - INFO - SciEDPipeR Memory benchmark::SciEDPipeR Memory: 35.58 KB SciEDPipeR Resident Memory: 9.66 KB SciEDPipeR Stack Size: 132.00 B + 2020-11-26 00:45:36,050 - FusionInspector - INFO - Pipeline.func_paths_are_from_valid_run: Not yet created without error. PATH=/data/FusionInspector/FusionInspector.consolidated.cSorted.bam + 2020-11-26 00:45:36,050 - FusionInspector - INFO - Pipeline.func_mkdirs: Did not create the following directory because it already exists /data/FusionInspector + 2020-11-26 00:45:36,050 - FusionInspector - INFO - Pipeline.func_run_commands: start command line: /opt/FusionInspector-v1.0.1/util/consolidate_bams_and_uniq_reads.pl /data/FusionInspector/fi_workdir/FusionInspector.fa /data/FusionInspector/fi_workdir/FusionInspector.star2.cSorted.rmdup.bam /data/FusionInspector/FusionInspector.consolidated + CMD: samtools view /data/FusionInspector/fi_workdir/FusionInspector.star2.cSorted.rmdup.bam >> /data/FusionInspector/FusionInspector.consolidated.tmp.sam + CMD: sort -k3,3 -k1,1 -k4,4n /data/FusionInspector/FusionInspector.consolidated.tmp.sam > /data/FusionInspector/FusionInspector.consolidated.tmp.sam.nameSorted + CMD: bash -c "set -o pipefail && cat /data/FusionInspector/FusionInspector.consolidated.tmp.sam.nameSorted.uniq | samtools view -Sb -T /data/FusionInspector/fi_workdir/FusionInspector.fa - 2>/dev/null | samtools sort - -o /data/FusionInspector/FusionInspector.consolidated.cSorted.bam" + CMD: samtools index /data/FusionInspector/FusionInspector.consolidated.cSorted.bam + 2020-11-26 00:45:36,355 - FusionInspector - INFO - Pipeline.func_run_commands: end commandline. + 2020-11-26 00:45:36,355 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:45:36,355 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.consolidated.cSorted.bam was made. + 2020-11-26 00:45:36,355 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:45:36,356 - FusionInspector - INFO - DependencyTree.func_complete_command: Products are made + 2020-11-26 00:45:36,356 - FusionInspector - INFO - DependencyTree.func_complete_command: /data/FusionInspector/FusionInspector.consolidated.cSorted.bam 242.43 KB + 2020-11-26 00:45:36,356 - FusionInspector - INFO - Pipeline.func_run_commands: Updated dependencies. True + 2020-11-26 00:45:36,356 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:45:36,356 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.consolidated.cSorted.bam was made. + 2020-11-26 00:45:36,356 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:45:36,356 - FusionInspector - INFO - Pipeline.func_run_commands: Time:: 0.0 + 2020-11-26 00:45:36,357 - FusionInspector - INFO - + + Pipeline.func_run_commands: Starting ln -s /data/FusionInspector/FusionInspector.consolidated.cSorted.bam /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam + 2020-11-26 00:45:36,357 - FusionInspector - INFO - SciEDPipeR Memory benchmark::SciEDPipeR Memory: 35.58 KB SciEDPipeR Resident Memory: 9.66 KB SciEDPipeR Stack Size: 132.00 B + 2020-11-26 00:45:36,357 - FusionInspector - INFO - Pipeline.func_paths_are_from_valid_run: Not yet created without error. PATH=/data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam + 2020-11-26 00:45:36,357 - FusionInspector - INFO - Pipeline.func_mkdirs: Did not create the following directory because it already exists /data/FusionInspector/fi_workdir + 2020-11-26 00:45:36,358 - FusionInspector - INFO - Pipeline.func_run_commands: start command line: ln -s /data/FusionInspector/FusionInspector.consolidated.cSorted.bam /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam + 2020-11-26 00:45:36,361 - FusionInspector - INFO - Pipeline.func_run_commands: end commandline. + 2020-11-26 00:45:36,362 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:45:36,362 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam was made. + 2020-11-26 00:45:36,362 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:45:36,362 - FusionInspector - INFO - DependencyTree.func_complete_command: Products are made + 2020-11-26 00:45:36,362 - FusionInspector - INFO - DependencyTree.func_complete_command: /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam 242.43 KB + 2020-11-26 00:45:36,362 - FusionInspector - INFO - Pipeline.func_run_commands: Updated dependencies. True + 2020-11-26 00:45:36,362 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:45:36,362 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam was made. + 2020-11-26 00:45:36,362 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:45:36,363 - FusionInspector - INFO - Pipeline.func_run_commands: Time:: 0.0 + 2020-11-26 00:45:36,363 - FusionInspector - INFO - + + Pipeline.func_run_commands: Starting /opt/trinityrnaseq-Trinity-v2.4.0//Trinity --genome_guided_bam /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam --max_memory 20G --genome_guided_max_intron 1000000 --CPU 20 --min_contig_length 100 --output /data/FusionInspector/fi_workdir/trinity_GG + 2020-11-26 00:45:36,363 - FusionInspector - INFO - SciEDPipeR Memory benchmark::SciEDPipeR Memory: 35.58 KB SciEDPipeR Resident Memory: 9.66 KB SciEDPipeR Stack Size: 132.00 B + 2020-11-26 00:45:36,363 - FusionInspector - INFO - Pipeline.func_paths_are_from_valid_run: Not yet created without error. PATH=/data/FusionInspector/fi_workdir/trinity_GG/Trinity-GG.fasta + 2020-11-26 00:45:36,364 - FusionInspector - INFO - Pipeline.func_mkdirs: Created directory /data/FusionInspector/fi_workdir/trinity_GG + 2020-11-26 00:45:36,364 - FusionInspector - INFO - Pipeline.func_run_commands: start command line: /opt/trinityrnaseq-Trinity-v2.4.0//Trinity --genome_guided_bam /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam --max_memory 20G --genome_guided_max_intron 1000000 --CPU 20 --min_contig_length 100 --output /data/FusionInspector/fi_workdir/trinity_GG + Trinity version: Trinity-v2.4.0 + Thursday, November 26, 2020: 00:45:39 CMD: /opt/trinityrnaseq-Trinity-v2.4.0/util/support_scripts/ensure_coord_sorted_sam.pl /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam + ** NOTE: Latest version of Trinity is v2.11.0, and can be obtained at: + https://github.com/trinityrnaseq/trinityrnaseq/releases + + -appears to be a coordinate sorted bam file. ok. + samtools view: writing to standard output failed: Broken pipe + samtools view: error closing standard output: -1 + Thursday, November 26, 2020: 00:45:39 CMD: java -Xmx64m -XX:ParallelGCThreads=2 -jar /opt/trinityrnaseq-Trinity-v2.4.0/util/support_scripts/ExitTester.jar 0 + Thursday, November 26, 2020: 00:45:39 CMD: java -Xmx64m -XX:ParallelGCThreads=2 -jar /opt/trinityrnaseq-Trinity-v2.4.0/util/support_scripts/ExitTester.jar 1 + + + ---------------------------------------------------------------------------------- + -------------- Trinity Phase 1: Clustering of RNA-Seq Reads --------------------- + ---------------------------------------------------------------------------------- + + Thursday, November 26, 2020: 00:45:40 CMD: samtools index /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam + Thursday, November 26, 2020: 00:45:40 CMD: /opt/trinityrnaseq-Trinity-v2.4.0/util/support_scripts/prep_rnaseq_alignments_for_genome_assisted_assembly.pl --coord_sorted_SAM /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam -I 1000000 --sort_buffer 20G --CPU 20 + CMD: /opt/trinityrnaseq-Trinity-v2.4.0/util/support_scripts//SAM_to_frag_coords.pl --CPU 20 --sort_buffer 20G --sam /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam --min_insert_size 1 --max_insert_size 1000000 + -extracting read coordinates from /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam into /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam.read_coords + + CMD: touch /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam.read_coords.ok + CMD: /usr/bin/sort --parallel=20 -S20G -T . -k1,1 -k2,2 -k4,4n /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam.read_coords > /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam.read_coords.sort_by_readname + CMD: /usr/bin/sort --parallel=20 -S20G -T . -k1,1 -k3,3n /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam.frag_coords > /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam.frag_coords.coord_sorted + CMD: /opt/trinityrnaseq-Trinity-v2.4.0/util/support_scripts//fragment_coverage_writer.pl /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam.frag_coords > /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam.frag_coverage.wig + [1000 lines read] scaff:RP11-255H23.4--RPSAP58 lend:2056 rend:10231 CMD: touch /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam.frag_coverage.wig.ok + CMD: /opt/trinityrnaseq-Trinity-v2.4.0/util/support_scripts//define_coverage_partitions.pl /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam.frag_coverage.wig 1 + > /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam.minC1.gff + CMD: touch /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam.minC1.gff.ok + CMD: /opt/trinityrnaseq-Trinity-v2.4.0/util/support_scripts//extract_reads_per_partition.pl --partitions_gff /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam.minC1.gff --coord_sorted_SAM /data/FusionInspector/fi_workdir/FusionInspector.consolidated.bam --parts_per_directory 100 --min_reads_per_partition 10 + // parsing paritions. + [2] -writing to Dir_FusionInspector.consolidated.bam.minC1.gff/EIF3K--CYP39A1/0/1070_30146.trinity.reads + -writing to Dir_FusionInspector.consolidated.bam.minC1.gff/RP11-255H23.4--RPSAP58/0/1645_10266.trinity.reads + CMD: touch Dir_FusionInspector.consolidated.bam.minC1.gff.ok + ## + Done + ## + + Thursday, November 26, 2020: 00:45:41 CMD: touch partitions.ok + Thursday, November 26, 2020: 00:45:41 CMD: find Dir_* -name '*reads' > read_files.list + Thursday, November 26, 2020: 00:45:41 CMD: touch read_files.list.ok + Thursday, November 26, 2020: 00:45:41 CMD: /opt/trinityrnaseq-Trinity-v2.4.0/util/support_scripts/write_partitioned_trinity_cmds.pl --reads_list_file read_files.list --CPU 1 --max_memory 1G --seqType fa --trinity_complete --full_cleanup --min_contig_length 100 > trinity_GG.cmds + Thursday, November 26, 2020: 00:45:41 CMD: touch trinity_GG.cmds.ok + Thursday, November 26, 2020: 00:45:41 CMD: touch trinity_GG.cmds.ok + + + -------------------------------------------------------------------------------- + ------------ Trinity Phase 2: Assembling Clusters of Reads --------------------- + -------------------------------------------------------------------------------- + + Thursday, November 26, 2020: 00:45:41 CMD: /opt/trinityrnaseq-Trinity-v2.4.0/trinity-plugins/parafly/bin/ParaFly -c trinity_GG.cmds -CPU 20 -v + Number of Commands: 2 + succeeded(1) 50% completed. succeeded(2) 100% completed. + + All commands completed successfully. :-) + + Thursday, November 26, 2020: 00:45:46 CMD: find Dir_* -name '*inity.fasta' | /opt/trinityrnaseq-Trinity-v2.4.0/util/support_scripts/GG_partitioned_trinity_aggregator.pl TRINITY_GG > Trinity-GG.fasta.tmp + + + Finished. See Trinity-GG.fasta for reconstructed transcripts + + 2020-11-26 00:45:46,224 - FusionInspector - INFO - Pipeline.func_run_commands: end commandline. + 2020-11-26 00:45:46,225 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:45:46,225 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/fi_workdir/trinity_GG/Trinity-GG.fasta was made. + 2020-11-26 00:45:46,225 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:45:46,225 - FusionInspector - INFO - DependencyTree.func_complete_command: Products are made + 2020-11-26 00:45:46,225 - FusionInspector - INFO - DependencyTree.func_complete_command: /data/FusionInspector/fi_workdir/trinity_GG/Trinity-GG.fasta 5.72 KB + 2020-11-26 00:45:46,225 - FusionInspector - INFO - Pipeline.func_run_commands: Updated dependencies. True + 2020-11-26 00:45:46,226 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:45:46,226 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/fi_workdir/trinity_GG/Trinity-GG.fasta was made. + 2020-11-26 00:45:46,226 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:45:46,226 - FusionInspector - INFO - Pipeline.func_run_commands: Time:: 10.0 + 2020-11-26 00:45:46,226 - FusionInspector - INFO - + + Pipeline.func_run_commands: Starting /opt/trinityrnaseq-Trinity-v2.4.0//util/misc/process_GMAP_alignments_gff3_chimeras_ok.pl --genome /data/FusionInspector/fi_workdir/FusionInspector.fa --transcripts /data/FusionInspector/fi_workdir/trinity_GG/Trinity-GG.fasta --no_chimera > /data/FusionInspector/fi_workdir/FusionInspector.gmap_trinity_GG.gff3 + 2020-11-26 00:45:46,227 - FusionInspector - INFO - SciEDPipeR Memory benchmark::SciEDPipeR Memory: 35.58 KB SciEDPipeR Resident Memory: 9.66 KB SciEDPipeR Stack Size: 132.00 B + 2020-11-26 00:45:46,227 - FusionInspector - INFO - Pipeline.func_paths_are_from_valid_run: Not yet created without error. PATH=/data/FusionInspector/fi_workdir/FusionInspector.gmap_trinity_GG.gff3 + 2020-11-26 00:45:46,227 - FusionInspector - INFO - Pipeline.func_mkdirs: Did not create the following directory because it already exists /data/FusionInspector/fi_workdir + 2020-11-26 00:45:46,228 - FusionInspector - INFO - Pipeline.func_run_commands: start command line: /opt/trinityrnaseq-Trinity-v2.4.0//util/misc/process_GMAP_alignments_gff3_chimeras_ok.pl --genome /data/FusionInspector/fi_workdir/FusionInspector.fa --transcripts /data/FusionInspector/fi_workdir/trinity_GG/Trinity-GG.fasta --no_chimera > /data/FusionInspector/fi_workdir/FusionInspector.gmap_trinity_GG.gff3 + CMD: gmap_build -D /data/FusionInspector/fi_workdir -T /data/FusionInspector/fi_workdir -d FusionInspector.fa.gmap -k 13 /data/FusionInspector/fi_workdir/FusionInspector.fa >&2 + Note: the -T flag is no longer necessary, since gmap_build now builds directly in the destination directory + Sorting chromosomes in chrom order. To turn off or sort other ways, use the -s flag. + Creating files in directory /data/FusionInspector/fi_workdir/FusionInspector.fa.gmap + Running "/usr/local/bin/fa_coords" -o "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap.coords" -f "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap.sources" + Opening file /data/FusionInspector/fi_workdir/FusionInspector.fa + Processed short contigs (<1000000 nt): .. + ============================================================ + Contig mapping information has been written to file /data/FusionInspector/fi_workdir/FusionInspector.fa.gmap.coords. + You should look at this file, and edit it if necessary + If everything is okay, you should proceed by running + make gmapdb + ============================================================ + Running "/usr/local/bin/gmap_process" -c "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap.coords" -f "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap.sources" | "/usr/local/bin/gmapindex" -d FusionInspector.fa.gmap -D "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap" -A + Reading coordinates from file /data/FusionInspector/fi_workdir/FusionInspector.fa.gmap.coords + Logging contig EIF3K--CYP39A1 at EIF3K--CYP39A1:1..31361 in genome FusionInspector.fa.gmap + => primary (linear) chromosome + Logging contig RP11-255H23.4--RPSAP58 at RP11-255H23.4--RPSAP58:1..11179 in genome FusionInspector.fa.gmap + => primary (linear) chromosome + Total genomic length = 42540 bp + Have a total of 2 chromosomes + Writing chromosome file /data/FusionInspector/fi_workdir/FusionInspector.fa.gmap/FusionInspector.fa.gmap.chromosome + Chromosome EIF3K--CYP39A1 has universal coordinates 1..31361 + Chromosome RP11-255H23.4--RPSAP58 has universal coordinates 31362..42540 + Writing chromosome IIT file /data/FusionInspector/fi_workdir/FusionInspector.fa.gmap/FusionInspector.fa.gmap.chromosome.iit + Writing IIT file header information...coordinates require 4 bytes each...done + Processing null division/chromosome...sorting...writing...done (2 intervals) + Writing IIT file footer information...done + Writing IIT file header information...coordinates require 4 bytes each...done + Processing null division/chromosome...sorting...writing...done (2 intervals) + Writing IIT file footer information...done + No alternate scaffolds observed + Running "/usr/local/bin/gmap_process" -c "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap.coords" -f "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap.sources" | "/usr/local/bin/gmapindex" -d FusionInspector.fa.gmap -F "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap" -D "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap" -G + Genome length is 42540 nt + Trying to allocate 3990*4 bytes of memory...succeeded. Building genome in memory. + Reading coordinates from file /data/FusionInspector/fi_workdir/FusionInspector.fa.gmap.coords + Writing contig EIF3K--CYP39A1 to universal coordinates 1..31361 + Writing contig RP11-255H23.4--RPSAP58 to universal coordinates 31362..42540 + A total of 0 non-ACGTNX characters were seen in the genome. + Running cat "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap/FusionInspector.fa.gmap.genomecomp" | "/usr/local/bin/gmapindex" -d FusionInspector.fa.gmap -U > "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap/FusionInspector.fa.gmap.genomebits128" + Running cat "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap/FusionInspector.fa.gmap.genomecomp" | "/usr/local/bin/gmapindex" -k 13 -q 3 -d FusionInspector.fa.gmap -F "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap" -D "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap" -N + Counting positions in genome FusionInspector.fa.gmap (13 bp every 3 bp), position 0 + Number of offsets: 13335 => pages file not required + Running "/usr/local/bin/gmapindex" -k 13 -q 3 -d FusionInspector.fa.gmap -F "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap" -D "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap" -O "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap/FusionInspector.fa.gmap.genomecomp" + Offset compression types: bitpack64 + Allocating 1048576*1 bytes for packsizes + Allocating 1048576*8 bytes for bitpacks + Indexing offsets of oligomers in genome FusionInspector.fa.gmap (13 bp every 3 bp), position 0 + Writing 67108865 offsets compressed via bitpack64...done + Running "/usr/local/bin/gmapindex" -k 13 -q 3 -d FusionInspector.fa.gmap -F "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap" -D "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap" -P "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap/FusionInspector.fa.gmap.genomecomp" + Looking for index files in directory /data/FusionInspector/fi_workdir/FusionInspector.fa.gmap + Pointers file is FusionInspector.fa.gmap.ref133offsets64meta + Offsets file is FusionInspector.fa.gmap.ref133offsets64strm + Positions file is FusionInspector.fa.gmap.ref133positions + Expanding offsetsstrm into counters...done + Allocating 206128 bytes for counterstrm + Trying to allocate 13335*4 bytes of memory for positions...succeeded. Building positions in memory. + Indexing positions of oligomers in genome FusionInspector.fa.gmap (13 bp every 3 bp), position 0 + Writing 13335 genomic positions to file /data/FusionInspector/fi_workdir/FusionInspector.fa.gmap/FusionInspector.fa.gmap.ref133positions ... + done + Running "/usr/local/bin/gmapindex" -d FusionInspector.fa.gmap -F "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap" -D "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap" -S + Genome length is 42540 + Building suffix array + SACA_K called with n = 42541, K = 5, level 0 + SACA_K called with n = 11206, K = 0, level 1 + SACA_K called with n = 3512, K = 0, level 2 + For indexsize 12, occupied 38646/16777216 + Optimal indexsize = 12 + Running "/usr/local/bin/gmapindex" -d FusionInspector.fa.gmap -F "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap" -D "/data/FusionInspector/fi_workdir/FusionInspector.fa.gmap" -L + Building LCP array + Writing temporary file for rank...done + Writing temporary file for permuted sarray...done + Byte-coding: 41050 values < 255, 1491 exceptions >= 255 (3.5%) + Building DC array + Building child array + Byte-coding: 42398 values < 255, 143 exceptions >= 255 (0.3%) + Writing file /data/FusionInspector/fi_workdir/FusionInspector.fa.gmap/FusionInspector.fa.gmap.salcpchilddcdone + Found 1491 exceptions + CMD: gmap -D /data/FusionInspector/fi_workdir -d FusionInspector.fa.gmap /data/FusionInspector/fi_workdir/trinity_GG/Trinity-GG.fasta -f 3 -n 1 -x 50 -t 2 -B 5 + GMAP version 2016-11-07 called with args: gmap.avx2 -D /data/FusionInspector/fi_workdir -d FusionInspector.fa.gmap /data/FusionInspector/fi_workdir/trinity_GG/Trinity-GG.fasta -f 3 -n 1 -x 50 -t 2 -B 5 + Note: -n 1 will not report chimeric alignments. If you want a single alignment plus chimeras, use -n 0 instead. + Note: Batch mode 5 is now the same as batch mode 4. + Expansion of offsets is now controlled separately by --expand-offsets (default=1). + Checking compiler assumptions for SSE2: 6B8B4567 327B23C6 xor=59F066A1 + Checking compiler assumptions for SSE4.1: -103 -58 max=198 => compiler zero extends + Checking compiler options for SSE4.2: 6B8B4567 __builtin_clz=1 __builtin_ctz=0 _mm_popcnt_u32=17 __builtin_popcount=17 + Finished checking compiler assumptions + Allocating memory for compressed genome (oligos)...Attached new memory for /data/FusionInspector/fi_workdir/FusionInspector.fa.gmap/FusionInspector.fa.gmap.genomecomp...done (15,960 bytes, 0.00 sec) + Allocating memory for compressed genome (bits)...Attached new memory for /data/FusionInspector/fi_workdir/FusionInspector.fa.gmap/FusionInspector.fa.gmap.genomebits128...done (15,984 bytes, 0.00 sec) + Looking for index files in directory /data/FusionInspector/fi_workdir/FusionInspector.fa.gmap + Pointers file is FusionInspector.fa.gmap.ref133offsets64meta + Offsets file is FusionInspector.fa.gmap.ref133offsets64strm + Positions file is FusionInspector.fa.gmap.ref133positions + Offsets compression type: bitpack64 + Allocating memory for ref offset pointers, kmer 13, interval 3...Attached new memory for /data/FusionInspector/fi_workdir/FusionInspector.fa.gmap/FusionInspector.fa.gmap.ref133offsets64meta...done (8,388,624 bytes, 0.01 sec) + Allocating memory for ref offsets, kmer 13, interval 3...Attached new memory for /data/FusionInspector/fi_workdir/FusionInspector.fa.gmap/FusionInspector.fa.gmap.ref133offsets64strm...done (206,128 bytes, 0.00 sec) + Allocating memory for ref positions, kmer 13, interval 3...Attached new memory for /data/FusionInspector/fi_workdir/FusionInspector.fa.gmap/FusionInspector.fa.gmap.ref133positions...done (53,340 bytes, 0.00 sec) + Starting alignment + Processed 8 queries in 0.05 seconds (160.00 queries/sec) + Removed existing memory for shmid 131076 + Removed existing memory for shmid 98307 + Removed existing memory for shmid 65538 + Removed existing memory for shmid 0 + Removed existing memory for shmid 32769 + 2020-11-26 00:46:20,059 - FusionInspector - INFO - Pipeline.func_run_commands: end commandline. + 2020-11-26 00:46:20,059 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,059 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/fi_workdir/FusionInspector.gmap_trinity_GG.gff3 was made. + 2020-11-26 00:46:20,060 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,060 - FusionInspector - INFO - DependencyTree.func_complete_command: Products are made + 2020-11-26 00:46:20,060 - FusionInspector - INFO - DependencyTree.func_complete_command: /data/FusionInspector/fi_workdir/FusionInspector.gmap_trinity_GG.gff3 5.33 KB + 2020-11-26 00:46:20,060 - FusionInspector - INFO - Pipeline.func_run_commands: Updated dependencies. True + 2020-11-26 00:46:20,060 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,060 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/fi_workdir/FusionInspector.gmap_trinity_GG.gff3 was made. + 2020-11-26 00:46:20,060 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,060 - FusionInspector - INFO - Pipeline.func_run_commands: Time:: 34.0 + 2020-11-26 00:46:20,061 - FusionInspector - INFO - + + Pipeline.func_run_commands: Starting /opt/FusionInspector-v1.0.1/util/get_Trinity_fusion_alignments_from_gff3.pl /data/FusionInspector/fi_workdir/FusionInspector.gtf /data/FusionInspector/fi_workdir/FusionInspector.gmap_trinity_GG.gff3 > /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3 + 2020-11-26 00:46:20,061 - FusionInspector - INFO - SciEDPipeR Memory benchmark::SciEDPipeR Memory: 35.58 KB SciEDPipeR Resident Memory: 9.66 KB SciEDPipeR Stack Size: 132.00 B + 2020-11-26 00:46:20,061 - FusionInspector - INFO - Pipeline.func_paths_are_from_valid_run: Not yet created without error. PATH=/data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3 + 2020-11-26 00:46:20,061 - FusionInspector - INFO - Pipeline.func_mkdirs: Did not create the following directory because it already exists /data/FusionInspector + 2020-11-26 00:46:20,061 - FusionInspector - INFO - Pipeline.func_run_commands: start command line: /opt/FusionInspector-v1.0.1/util/get_Trinity_fusion_alignments_from_gff3.pl /data/FusionInspector/fi_workdir/FusionInspector.gtf /data/FusionInspector/fi_workdir/FusionInspector.gmap_trinity_GG.gff3 > /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3 + 2020-11-26 00:46:20,104 - FusionInspector - INFO - Pipeline.func_run_commands: end commandline. + 2020-11-26 00:46:20,104 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,104 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3 was made. + 2020-11-26 00:46:20,104 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,104 - FusionInspector - INFO - DependencyTree.func_complete_command: Products are made + 2020-11-26 00:46:20,105 - FusionInspector - INFO - DependencyTree.func_complete_command: /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3 762.0 B + 2020-11-26 00:46:20,105 - FusionInspector - INFO - Pipeline.func_run_commands: Updated dependencies. True + 2020-11-26 00:46:20,105 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,105 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3 was made. + 2020-11-26 00:46:20,105 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,105 - FusionInspector - INFO - Pipeline.func_run_commands: Time:: 0.0 + 2020-11-26 00:46:20,106 - FusionInspector - INFO - + + Pipeline.func_run_commands: Starting /opt/FusionInspector-v1.0.1/util/get_Trinity_fusion_fasta_seqs.pl /data/FusionInspector/fi_workdir/trinity_GG/Trinity-GG.fasta /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3 > /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.fasta + 2020-11-26 00:46:20,106 - FusionInspector - INFO - SciEDPipeR Memory benchmark::SciEDPipeR Memory: 35.58 KB SciEDPipeR Resident Memory: 9.66 KB SciEDPipeR Stack Size: 132.00 B + 2020-11-26 00:46:20,106 - FusionInspector - INFO - Pipeline.func_paths_are_from_valid_run: Not yet created without error. PATH=/data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.fasta + 2020-11-26 00:46:20,107 - FusionInspector - INFO - Pipeline.func_mkdirs: Did not create the following directory because it already exists /data/FusionInspector + 2020-11-26 00:46:20,107 - FusionInspector - INFO - Pipeline.func_run_commands: start command line: /opt/FusionInspector-v1.0.1/util/get_Trinity_fusion_fasta_seqs.pl /data/FusionInspector/fi_workdir/trinity_GG/Trinity-GG.fasta /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3 > /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.fasta + 2020-11-26 00:46:20,136 - FusionInspector - INFO - Pipeline.func_run_commands: end commandline. + 2020-11-26 00:46:20,136 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,137 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.fasta was made. + 2020-11-26 00:46:20,137 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,137 - FusionInspector - INFO - DependencyTree.func_complete_command: Products are made + 2020-11-26 00:46:20,137 - FusionInspector - INFO - DependencyTree.func_complete_command: /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.fasta 434.0 B + 2020-11-26 00:46:20,137 - FusionInspector - INFO - Pipeline.func_run_commands: Updated dependencies. True + 2020-11-26 00:46:20,137 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,138 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.fasta was made. + 2020-11-26 00:46:20,138 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,138 - FusionInspector - INFO - Pipeline.func_run_commands: Time:: 0.0 + 2020-11-26 00:46:20,138 - FusionInspector - INFO - + + Pipeline.func_run_commands: Starting /opt/FusionInspector-v1.0.1/util/transcript_gff3_to_bed.pl /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3 95 > /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed + 2020-11-26 00:46:20,139 - FusionInspector - INFO - SciEDPipeR Memory benchmark::SciEDPipeR Memory: 35.58 KB SciEDPipeR Resident Memory: 9.66 KB SciEDPipeR Stack Size: 132.00 B + 2020-11-26 00:46:20,139 - FusionInspector - INFO - Pipeline.func_paths_are_from_valid_run: Not yet created without error. PATH=/data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed + 2020-11-26 00:46:20,139 - FusionInspector - INFO - Pipeline.func_mkdirs: Did not create the following directory because it already exists /data/FusionInspector + 2020-11-26 00:46:20,139 - FusionInspector - INFO - Pipeline.func_run_commands: start command line: /opt/FusionInspector-v1.0.1/util/transcript_gff3_to_bed.pl /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3 95 > /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed + ignoring line: #TrinityFusionTranscript: TRINITY_GG_1_c2_g1_i1 EIF3K--CYP39A1:8757-17360 + 2020-11-26 00:46:20,200 - FusionInspector - INFO - Pipeline.func_run_commands: end commandline. + 2020-11-26 00:46:20,200 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,200 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed was made. + 2020-11-26 00:46:20,200 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,200 - FusionInspector - INFO - DependencyTree.func_complete_command: Products are made + 2020-11-26 00:46:20,201 - FusionInspector - INFO - DependencyTree.func_complete_command: /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed 182.0 B + 2020-11-26 00:46:20,201 - FusionInspector - INFO - Pipeline.func_run_commands: Updated dependencies. True + 2020-11-26 00:46:20,201 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,201 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed was made. + 2020-11-26 00:46:20,201 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,201 - FusionInspector - INFO - Pipeline.func_run_commands: Time:: 0.0 + 2020-11-26 00:46:20,202 - FusionInspector - INFO - + + Pipeline.func_run_commands: Starting sort -k1,1 -k2,2n /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed > /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed.sorted.bed + 2020-11-26 00:46:20,202 - FusionInspector - INFO - SciEDPipeR Memory benchmark::SciEDPipeR Memory: 35.58 KB SciEDPipeR Resident Memory: 9.66 KB SciEDPipeR Stack Size: 132.00 B + 2020-11-26 00:46:20,202 - FusionInspector - INFO - Pipeline.func_paths_are_from_valid_run: Not yet created without error. PATH=/data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed.sorted.bed + 2020-11-26 00:46:20,203 - FusionInspector - INFO - Pipeline.func_mkdirs: Did not create the following directory because it already exists /data/FusionInspector + 2020-11-26 00:46:20,203 - FusionInspector - INFO - Pipeline.func_run_commands: start command line: sort -k1,1 -k2,2n /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed > /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed.sorted.bed + 2020-11-26 00:46:20,208 - FusionInspector - INFO - Pipeline.func_run_commands: end commandline. + 2020-11-26 00:46:20,209 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,209 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed.sorted.bed was made. + 2020-11-26 00:46:20,209 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,209 - FusionInspector - INFO - DependencyTree.func_complete_command: Products are made + 2020-11-26 00:46:20,209 - FusionInspector - INFO - DependencyTree.func_complete_command: /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed.sorted.bed 182.0 B + 2020-11-26 00:46:20,210 - FusionInspector - INFO - Pipeline.func_run_commands: Updated dependencies. True + 2020-11-26 00:46:20,210 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,210 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed.sorted.bed was made. + 2020-11-26 00:46:20,210 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,210 - FusionInspector - INFO - Pipeline.func_run_commands: Time:: 0.0 + 2020-11-26 00:46:20,210 - FusionInspector - INFO - + + Pipeline.func_run_commands: Starting bgzip -f /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed.sorted.bed + 2020-11-26 00:46:20,211 - FusionInspector - INFO - SciEDPipeR Memory benchmark::SciEDPipeR Memory: 35.58 KB SciEDPipeR Resident Memory: 9.67 KB SciEDPipeR Stack Size: 132.00 B + 2020-11-26 00:46:20,211 - FusionInspector - INFO - Pipeline.func_paths_are_from_valid_run: Not yet created without error. PATH=/data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed.sorted.bed.gz + 2020-11-26 00:46:20,211 - FusionInspector - INFO - Pipeline.func_mkdirs: Did not create the following directory because it already exists /data/FusionInspector + 2020-11-26 00:46:20,211 - FusionInspector - INFO - Pipeline.func_run_commands: start command line: bgzip -f /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed.sorted.bed + 2020-11-26 00:46:20,217 - FusionInspector - INFO - Pipeline.func_run_commands: end commandline. + 2020-11-26 00:46:20,218 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,218 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed.sorted.bed.gz was made. + 2020-11-26 00:46:20,218 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,218 - FusionInspector - INFO - DependencyTree.func_complete_command: Products are made + 2020-11-26 00:46:20,218 - FusionInspector - INFO - DependencyTree.func_complete_command: /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed.sorted.bed.gz 163.0 B + 2020-11-26 00:46:20,219 - FusionInspector - INFO - Pipeline.func_run_commands: Updated dependencies. True + 2020-11-26 00:46:20,219 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,219 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed.sorted.bed.gz was made. + 2020-11-26 00:46:20,219 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,219 - FusionInspector - INFO - Pipeline.func_run_commands: Time:: 0.0 + 2020-11-26 00:46:20,220 - FusionInspector - INFO - + + Pipeline.func_run_commands: Starting tabix -p bed /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed.sorted.bed.gz + 2020-11-26 00:46:20,220 - FusionInspector - INFO - SciEDPipeR Memory benchmark::SciEDPipeR Memory: 35.58 KB SciEDPipeR Resident Memory: 9.67 KB SciEDPipeR Stack Size: 132.00 B + 2020-11-26 00:46:20,221 - FusionInspector - INFO - Pipeline.func_paths_are_from_valid_run: Not yet created without error. PATH=/data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed.sorted.bed.gz.tbi + 2020-11-26 00:46:20,221 - FusionInspector - INFO - Pipeline.func_mkdirs: Did not create the following directory because it already exists /data/FusionInspector + 2020-11-26 00:46:20,221 - FusionInspector - INFO - Pipeline.func_run_commands: start command line: tabix -p bed /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed.sorted.bed.gz + 2020-11-26 00:46:20,227 - FusionInspector - INFO - Pipeline.func_run_commands: end commandline. + 2020-11-26 00:46:20,227 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,227 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed.sorted.bed.gz.tbi was made. + 2020-11-26 00:46:20,227 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,228 - FusionInspector - INFO - DependencyTree.func_complete_command: Products are made + 2020-11-26 00:46:20,228 - FusionInspector - INFO - DependencyTree.func_complete_command: /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed.sorted.bed.gz.tbi 116.0 B + 2020-11-26 00:46:20,228 - FusionInspector - INFO - Pipeline.func_run_commands: Updated dependencies. True + 2020-11-26 00:46:20,228 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,228 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3.bed.sorted.bed.gz.tbi was made. + 2020-11-26 00:46:20,228 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,229 - FusionInspector - INFO - Pipeline.func_run_commands: Time:: 0.0 + 2020-11-26 00:46:20,229 - FusionInspector - INFO - + + Pipeline.func_run_commands: Starting /opt/FusionInspector-v1.0.1/util/add_TrinityGG_to_fusion_summary.pl /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3 > /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG + 2020-11-26 00:46:20,229 - FusionInspector - INFO - SciEDPipeR Memory benchmark::SciEDPipeR Memory: 35.58 KB SciEDPipeR Resident Memory: 9.67 KB SciEDPipeR Stack Size: 132.00 B + 2020-11-26 00:46:20,229 - FusionInspector - INFO - Pipeline.func_paths_are_from_valid_run: Not yet created without error. PATH=/data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG + 2020-11-26 00:46:20,230 - FusionInspector - INFO - Pipeline.func_mkdirs: Did not create the following directory because it already exists /data/FusionInspector/fi_workdir + 2020-11-26 00:46:20,230 - FusionInspector - INFO - Pipeline.func_run_commands: start command line: /opt/FusionInspector-v1.0.1/util/add_TrinityGG_to_fusion_summary.pl /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh /data/FusionInspector/FusionInspector.gmap_trinity_GG.fusions.gff3 > /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG + 2020-11-26 00:46:20,241 - FusionInspector - INFO - Pipeline.func_run_commands: end commandline. + 2020-11-26 00:46:20,241 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,241 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG was made. + 2020-11-26 00:46:20,241 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,241 - FusionInspector - INFO - DependencyTree.func_complete_command: Products are made + 2020-11-26 00:46:20,242 - FusionInspector - INFO - DependencyTree.func_complete_command: /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG 3.07 KB + 2020-11-26 00:46:20,242 - FusionInspector - INFO - Pipeline.func_run_commands: Updated dependencies. True + 2020-11-26 00:46:20,242 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,242 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG was made. + 2020-11-26 00:46:20,242 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,242 - FusionInspector - INFO - Pipeline.func_run_commands: Time:: 0.0 + 2020-11-26 00:46:20,242 - FusionInspector - INFO - + + Pipeline.func_run_commands: Starting /opt/FusionInspector-v1.0.1/util/starfmt_reformatter.pl --fusion_preds /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG > /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG.starFfmt + 2020-11-26 00:46:20,243 - FusionInspector - INFO - SciEDPipeR Memory benchmark::SciEDPipeR Memory: 35.58 KB SciEDPipeR Resident Memory: 9.67 KB SciEDPipeR Stack Size: 132.00 B + 2020-11-26 00:46:20,243 - FusionInspector - INFO - Pipeline.func_paths_are_from_valid_run: Not yet created without error. PATH=/data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG.starFfmt + 2020-11-26 00:46:20,243 - FusionInspector - INFO - Pipeline.func_mkdirs: Did not create the following directory because it already exists /data/FusionInspector/fi_workdir + 2020-11-26 00:46:20,243 - FusionInspector - INFO - Pipeline.func_run_commands: start command line: /opt/FusionInspector-v1.0.1/util/starfmt_reformatter.pl --fusion_preds /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG > /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG.starFfmt + 2020-11-26 00:46:20,284 - FusionInspector - INFO - Pipeline.func_run_commands: end commandline. + 2020-11-26 00:46:20,284 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,284 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG.starFfmt was made. + 2020-11-26 00:46:20,285 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,285 - FusionInspector - INFO - DependencyTree.func_complete_command: Products are made + 2020-11-26 00:46:20,285 - FusionInspector - INFO - DependencyTree.func_complete_command: /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG.starFfmt 3.1 KB + 2020-11-26 00:46:20,285 - FusionInspector - INFO - Pipeline.func_run_commands: Updated dependencies. True + 2020-11-26 00:46:20,285 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,285 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG.starFfmt was made. + 2020-11-26 00:46:20,285 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,286 - FusionInspector - INFO - Pipeline.func_run_commands: Time:: 0.0 + 2020-11-26 00:46:20,286 - FusionInspector - INFO - + + Pipeline.func_run_commands: Starting /opt/FusionInspector-v1.0.1/util/append_breakpoint_junction_info_via_FI_contigs.pl /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG.starFfmt /data/FusionInspector/FusionInspector.fa > /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG.starFfmt.wSpliceInfo + 2020-11-26 00:46:20,286 - FusionInspector - INFO - SciEDPipeR Memory benchmark::SciEDPipeR Memory: 35.58 KB SciEDPipeR Resident Memory: 9.67 KB SciEDPipeR Stack Size: 132.00 B + 2020-11-26 00:46:20,286 - FusionInspector - INFO - Pipeline.func_paths_are_from_valid_run: Not yet created without error. PATH=/data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG.starFfmt.wSpliceInfo + 2020-11-26 00:46:20,287 - FusionInspector - INFO - Pipeline.func_mkdirs: Did not create the following directory because it already exists /data/FusionInspector/fi_workdir + 2020-11-26 00:46:20,287 - FusionInspector - INFO - Pipeline.func_run_commands: start command line: /opt/FusionInspector-v1.0.1/util/append_breakpoint_junction_info_via_FI_contigs.pl /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG.starFfmt /data/FusionInspector/FusionInspector.fa > /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG.starFfmt.wSpliceInfo + 2020-11-26 00:46:20,319 - FusionInspector - INFO - Pipeline.func_run_commands: end commandline. + 2020-11-26 00:46:20,319 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,320 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG.starFfmt.wSpliceInfo was made. + 2020-11-26 00:46:20,320 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,320 - FusionInspector - INFO - DependencyTree.func_complete_command: Products are made + 2020-11-26 00:46:20,320 - FusionInspector - INFO - DependencyTree.func_complete_command: /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG.starFfmt.wSpliceInfo 3.18 KB + 2020-11-26 00:46:20,320 - FusionInspector - INFO - Pipeline.func_run_commands: Updated dependencies. True + 2020-11-26 00:46:20,320 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,320 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG.starFfmt.wSpliceInfo was made. + 2020-11-26 00:46:20,321 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,321 - FusionInspector - INFO - Pipeline.func_run_commands: Time:: 0.0 + 2020-11-26 00:46:20,321 - FusionInspector - INFO - + + Pipeline.func_run_commands: Starting /opt/FusionInspector-v1.0.1/util/../FusionFilter/blast_and_promiscuity_filter.pl --fusion_preds /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG.starFfmt.wSpliceInfo --out_prefix /data/FusionInspector/FusionInspector.fusion_predictions --genome_lib_dir star_with_fusion_100bp_readlen_index --max_promiscuity 3 -E 0.001 + 2020-11-26 00:46:20,322 - FusionInspector - INFO - SciEDPipeR Memory benchmark::SciEDPipeR Memory: 35.58 KB SciEDPipeR Resident Memory: 9.67 KB SciEDPipeR Stack Size: 132.00 B + 2020-11-26 00:46:20,322 - FusionInspector - INFO - Pipeline.func_paths_are_from_valid_run: Not yet created without error. PATH=/data/FusionInspector/FusionInspector.fusion_predictions.final + 2020-11-26 00:46:20,322 - FusionInspector - INFO - Pipeline.func_mkdirs: Did not create the following directory because it already exists /data/FusionInspector + 2020-11-26 00:46:20,322 - FusionInspector - INFO - Pipeline.func_run_commands: start command line: /opt/FusionInspector-v1.0.1/util/../FusionFilter/blast_and_promiscuity_filter.pl --fusion_preds /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG.starFfmt.wSpliceInfo --out_prefix /data/FusionInspector/FusionInspector.fusion_predictions --genome_lib_dir star_with_fusion_100bp_readlen_index --max_promiscuity 3 -E 0.001 + CMD: /opt/FusionInspector-v1.0.1/FusionFilter/util/blast_filter.pl --fusion_preds /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG.starFfmt.wSpliceInfo -E 0.001 --genome_lib_dir star_with_fusion_100bp_readlen_index + CMD: /opt/FusionInspector-v1.0.1/FusionFilter/util/promiscuity_filter.pl --fusion_preds /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG.starFfmt.wSpliceInfo.post_blast_filter --genome_lib_dir star_with_fusion_100bp_readlen_index --max_promiscuity 3 --min_pct_dom_promiscuity 20 + CMD: cp /data/FusionInspector/fi_workdir/FusionInspector.fusion_preds.coalesced.summary.min_frag_thresh.wTrinityGG.starFfmt.wSpliceInfo.post_blast_filter.post_promisc_filter /data/FusionInspector/FusionInspector.fusion_predictions.final + 2020-11-26 00:46:20,947 - FusionInspector - INFO - Pipeline.func_run_commands: end commandline. + 2020-11-26 00:46:20,948 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,948 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.fusion_predictions.final was made. + 2020-11-26 00:46:20,948 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,948 - FusionInspector - INFO - DependencyTree.func_complete_command: Products are made + 2020-11-26 00:46:20,948 - FusionInspector - INFO - DependencyTree.func_complete_command: /data/FusionInspector/FusionInspector.fusion_predictions.final 3.18 KB + 2020-11-26 00:46:20,948 - FusionInspector - INFO - Pipeline.func_run_commands: Updated dependencies. True + 2020-11-26 00:46:20,949 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,949 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.fusion_predictions.final was made. + 2020-11-26 00:46:20,949 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,949 - FusionInspector - INFO - Pipeline.func_run_commands: Time:: 1.0 + 2020-11-26 00:46:20,949 - FusionInspector - INFO - + + Pipeline.func_run_commands: Starting /opt/FusionInspector-v1.0.1/util/column_exclusions.pl /data/FusionInspector/FusionInspector.fusion_predictions.final JunctionReads,SpanningFrags,CounterFusionLeftReads,CounterFusionRightReads > /data/FusionInspector/FusionInspector.fusion_predictions.final.abridged + 2020-11-26 00:46:20,950 - FusionInspector - INFO - SciEDPipeR Memory benchmark::SciEDPipeR Memory: 35.58 KB SciEDPipeR Resident Memory: 9.67 KB SciEDPipeR Stack Size: 132.00 B + 2020-11-26 00:46:20,950 - FusionInspector - INFO - Pipeline.func_paths_are_from_valid_run: Not yet created without error. PATH=/data/FusionInspector/FusionInspector.fusion_predictions.final.abridged + 2020-11-26 00:46:20,950 - FusionInspector - INFO - Pipeline.func_mkdirs: Did not create the following directory because it already exists /data/FusionInspector + 2020-11-26 00:46:20,951 - FusionInspector - INFO - Pipeline.func_run_commands: start command line: /opt/FusionInspector-v1.0.1/util/column_exclusions.pl /data/FusionInspector/FusionInspector.fusion_predictions.final JunctionReads,SpanningFrags,CounterFusionLeftReads,CounterFusionRightReads > /data/FusionInspector/FusionInspector.fusion_predictions.final.abridged + 2020-11-26 00:46:20,983 - FusionInspector - INFO - Pipeline.func_run_commands: end commandline. + 2020-11-26 00:46:20,984 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,984 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.fusion_predictions.final.abridged was made. + 2020-11-26 00:46:20,984 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,984 - FusionInspector - INFO - DependencyTree.func_complete_command: Products are made + 2020-11-26 00:46:20,984 - FusionInspector - INFO - DependencyTree.func_complete_command: /data/FusionInspector/FusionInspector.fusion_predictions.final.abridged 453.0 B + 2020-11-26 00:46:20,984 - FusionInspector - INFO - Pipeline.func_run_commands: Updated dependencies. True + 2020-11-26 00:46:20,984 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:20,985 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.fusion_predictions.final.abridged was made. + 2020-11-26 00:46:20,985 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:20,985 - FusionInspector - INFO - Pipeline.func_run_commands: Time:: 0.0 + 2020-11-26 00:46:20,985 - FusionInspector - INFO - + + Pipeline.func_run_commands: Starting /opt/FusionInspector-v1.0.1/util/incorporate_FFPM_into_final_report.pl /data/rna_1.fq.gz /data/FusionInspector/FusionInspector.fusion_predictions.final.abridged > /data/FusionInspector/FusionInspector.fusion_predictions.final.abridged.FFPM + 2020-11-26 00:46:20,986 - FusionInspector - INFO - SciEDPipeR Memory benchmark::SciEDPipeR Memory: 35.58 KB SciEDPipeR Resident Memory: 9.67 KB SciEDPipeR Stack Size: 132.00 B + 2020-11-26 00:46:20,986 - FusionInspector - INFO - Pipeline.func_paths_are_from_valid_run: Not yet created without error. PATH=/data/FusionInspector/FusionInspector.fusion_predictions.final.abridged.FFPM + 2020-11-26 00:46:20,986 - FusionInspector - INFO - Pipeline.func_mkdirs: Did not create the following directory because it already exists /data/FusionInspector + 2020-11-26 00:46:20,986 - FusionInspector - INFO - Pipeline.func_run_commands: start command line: /opt/FusionInspector-v1.0.1/util/incorporate_FFPM_into_final_report.pl /data/rna_1.fq.gz /data/FusionInspector/FusionInspector.fusion_predictions.final.abridged > /data/FusionInspector/FusionInspector.fusion_predictions.final.abridged.FFPM + 2020-11-26 00:46:24,210 - FusionInspector - INFO - Pipeline.func_run_commands: end commandline. + 2020-11-26 00:46:24,211 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:24,211 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.fusion_predictions.final.abridged.FFPM was made. + 2020-11-26 00:46:24,211 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:24,211 - FusionInspector - INFO - DependencyTree.func_complete_command: Products are made + 2020-11-26 00:46:24,211 - FusionInspector - INFO - DependencyTree.func_complete_command: /data/FusionInspector/FusionInspector.fusion_predictions.final.abridged.FFPM 481.0 B + 2020-11-26 00:46:24,211 - FusionInspector - INFO - Pipeline.func_run_commands: Updated dependencies. True + 2020-11-26 00:46:24,212 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:24,212 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/FusionInspector.fusion_predictions.final.abridged.FFPM was made. + 2020-11-26 00:46:24,212 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:24,212 - FusionInspector - INFO - Pipeline.func_run_commands: Time:: 3.0 + 2020-11-26 00:46:24,212 - FusionInspector - INFO - + + Pipeline.func_run_commands: Starting /bin/rm -rf /data/FusionInspector/fi_workdir && touch /data/FusionInspector/workdir.cleaned + 2020-11-26 00:46:24,213 - FusionInspector - INFO - SciEDPipeR Memory benchmark::SciEDPipeR Memory: 35.58 KB SciEDPipeR Resident Memory: 9.67 KB SciEDPipeR Stack Size: 132.00 B + 2020-11-26 00:46:24,213 - FusionInspector - INFO - Pipeline.func_paths_are_from_valid_run: Not yet created without error. PATH=/data/FusionInspector/workdir.cleaned + 2020-11-26 00:46:24,213 - FusionInspector - INFO - Pipeline.func_mkdirs: Did not create the following directory because it already exists /data/FusionInspector + 2020-11-26 00:46:24,213 - FusionInspector - INFO - Pipeline.func_run_commands: start command line: /bin/rm -rf /data/FusionInspector/fi_workdir && touch /data/FusionInspector/workdir.cleaned + 2020-11-26 00:46:24,240 - FusionInspector - INFO - Pipeline.func_run_commands: end commandline. + 2020-11-26 00:46:24,241 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:24,241 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/workdir.cleaned was made. + 2020-11-26 00:46:24,241 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:24,241 - FusionInspector - INFO - DependencyTree.func_complete_command: Products are made + 2020-11-26 00:46:24,241 - FusionInspector - INFO - DependencyTree.func_complete_command: /data/FusionInspector/workdir.cleaned 0.0 B + 2020-11-26 00:46:24,241 - FusionInspector - INFO - Pipeline.func_run_commands: Updated dependencies. True + 2020-11-26 00:46:24,242 - FusionInspector - INFO - DependencyTree.func_paths_made: Currently at /data + 2020-11-26 00:46:24,242 - FusionInspector - INFO - DependencyTree.func_paths_made: Checking that /data/FusionInspector/workdir.cleaned was made. + 2020-11-26 00:46:24,242 - FusionInspector - INFO - DependencyTree.func_paths_made: Files were made. + 2020-11-26 00:46:24,242 - FusionInspector - INFO - Pipeline.func_run_commands: Time:: 0.0 + 2020-11-26 00:46:24,242 - FusionInspector - INFO - Pipeline.func_run_commands: Successfully ended pipeline FusionInspector + 2020-11-26 00:46:24,242 - FusionInspector - INFO - Pipeline.func_run_commands: Cleaning was not turned on. All files should be available. + [2020-11-25T16:46:25-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Exporting files/for-job/kind-run_fusion/instance-g9_082sr/file-r8tdbt36/FusionInspector.fusion_predictions.final.abridged.FFPM to output location + [2020-11-25T16:46:25-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Exporting files/for-job/kind-run_fusion/instance-g9_082sr/file-m3vqnlur/FusionInspector.gmap_trinity_GG.fusions.fasta to output location + [2020-11-25T16:46:25-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Exporting files/for-job/kind-run_fusion/instance-g9_082sr/file-1y9poi2h/FusionInspector.gmap_trinity_GG.fusions.gff3 to output location + [2020-11-25T16:46:25-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Exporting files/for-job/kind-run_fusion/instance-g9_082sr/file-al3fr6ki/fusion.final to output location + [2020-11-25T16:46:25-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Ran STAR-Fusion on TEST successfully + [2020-11-25T16:46:25-0800] [MainThread] [W] [toil.resource] Can't globalize module ModuleDescriptor(dirPath='/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages', name='toil.job', fromVirtualEnv=True). + [2020-11-25T16:46:25-0800] [MainThread] [W] [toil.resource] Can't globalize module ModuleDescriptor(dirPath='/private/home/drkthomp/protect3/src', name='protect.mutation_calling.fusion', fromVirtualEnv=False). + [2020-11-25T16:46:25-0800] [MainThread] [W] [toil.resource] Can't globalize module ModuleDescriptor(dirPath='/private/home/drkthomp/protect3/src', name='protect.mutation_calling.fusion', fromVirtualEnv=False). + [2020-11-25T16:46:25-0800] [MainThread] [W] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-v67rsjos/cleanup/file-d_nkrp7o/stream used 111.61% (59.6 GB [64029351936B] used, 53.4 GB [57369019908B] requested) at the end of its run. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/fusion.py", line 407, in reformat_star_fusion_output + transcripts) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/fusion.py", line 300, in split_fusion_transcript + trans = string.maketrans(forward, reverse) + AttributeError: module 'string' has no attribute 'maketrans' + [2020-11-25T16:46:25-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-25T16:46:34-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'reformat_star_fusion_output' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr to 1 +[2020-11-25T16:46:35-0800] [MainThread] [I] [toil.leader] Issued job 'reformat_star_fusion_output' kind-run_fusion/instance-g9_082sr with job batch system ID: 27 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-25T16:46:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmppk9vilu7/worker_log.txt +[2020-11-25T16:51:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-25-2020 16:51:03: Exporting files/for-job/kind-run_rsem/instance-mzzok2e3/file-3slg5vmy/rsem.genes.results to output location +[2020-11-25T16:51:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-25-2020 16:51:03: Exporting files/for-job/kind-run_rsem/instance-mzzok2e3/file-7xt9p03w/rsem.isoforms.results to output location +[2020-11-25T16:51:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-25-2020 16:51:03: Ran rsem on TEST successfully +[2020-11-25T16:51:04-0800] [MainThread] [I] [toil.leader] Job ended: 'run_rsem' kind-run_rsem/instance-mzzok2e3 +[2020-11-25T16:51:04-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-xfyn9sg_ with job batch system ID: 28 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-25T16:51:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpsu9fjld6/worker_log.txt +[2020-11-25T16:51:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp__udvypa/worker_log.txt +[2020-11-25T16:51:05-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker reformat_star_fusion_output file:/scratch/drkthomp/jobStore kind-run_fusion/instance-g9_082sr. +[2020-11-25T16:51:05-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'reformat_star_fusion_output' kind-run_fusion/instance-g9_082sr +[2020-11-25T16:51:05-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'reformat_star_fusion_output' kind-run_fusion/instance-g9_082sr +[2020-11-25T16:51:05-0800] [MainThread] [W] [toil.leader] Log from job kind-run_fusion/instance-g9_082sr follows: +=========> + [2020-11-25T16:51:05-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-25T16:51:05-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/fusion.py", line 407, in reformat_star_fusion_output + transcripts) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/fusion.py", line 300, in split_fusion_transcript + trans = string.maketrans(forward, reverse) + AttributeError: module 'string' has no attribute 'maketrans' + [2020-11-25T16:51:05-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-25T16:51:05-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'reformat_star_fusion_output' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr to 0 +[2020-11-25T16:51:05-0800] [MainThread] [W] [toil.leader] Job 'reformat_star_fusion_output' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr is completely failed +[2020-11-25T16:51:05-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-xfyn9sg_ +[2020-11-25T16:51:05-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-yfabv7w5 with job batch system ID: 29 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-25T16:51:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmprmy2lnz7/worker_log.txt +[2020-11-25T16:51:06-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-yfabv7w5 +[2020-11-25T16:51:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mhc_gene_assessment' kind-run_mhc_gene_assessment/instance-qkfz94bs with job batch system ID: 30 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-25T16:51:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_itx_resistance_assessment' kind-run_itx_resistance_assessment/instance-00iiueib with job batch system ID: 31 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-25T16:51:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_car_t_validity_assessment' kind-run_car_t_validity_assessment/instance-uyx66yka with job batch system ID: 32 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-25T16:51:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpo_mefujm/worker_log.txt +[2020-11-25T16:51:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpw_q6h0ug/worker_log.txt +[2020-11-25T16:51:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpaenihtc9/worker_log.txt +[2020-11-25T16:51:07-0800] [MainThread] [I] [toil.leader] Job ended: 'run_car_t_validity_assessment' kind-run_car_t_validity_assessment/instance-uyx66yka +[2020-11-25T16:51:07-0800] [MainThread] [I] [toil.leader] Issued job 'assess_car_t_validity' kind-assess_car_t_validity/instance-ijje8btr with job batch system ID: 33 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-25T16:51:07-0800] [MainThread] [I] [toil.leader] Job ended: 'run_itx_resistance_assessment' kind-run_itx_resistance_assessment/instance-00iiueib +[2020-11-25T16:51:07-0800] [MainThread] [I] [toil.leader] Issued job 'assess_itx_resistance' kind-assess_itx_resistance/instance-a2s3kbby with job batch system ID: 34 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-25T16:51:07-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mhc_gene_assessment' kind-run_mhc_gene_assessment/instance-qkfz94bs +[2020-11-25T16:51:07-0800] [MainThread] [I] [toil.leader] Issued job 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c with job batch system ID: 35 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-25T16:51:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmppo93v2ci/worker_log.txt +[2020-11-25T16:51:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpxx5p7k25/worker_log.txt +[2020-11-25T16:51:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpob0rs17a/worker_log.txt +[2020-11-25T16:51:09-0800] [MainThread] [I] [toil.leader] Job ended: 'assess_itx_resistance' kind-assess_itx_resistance/instance-a2s3kbby +[2020-11-25T16:51:09-0800] [MainThread] [I] [toil.leader] Issued job 'run_itx_resistance_assessment' kind-run_itx_resistance_assessment/instance-00iiueib with job batch system ID: 36 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-25T16:51:09-0800] [MainThread] [I] [toil.leader] Job ended: 'assess_car_t_validity' kind-assess_car_t_validity/instance-ijje8btr +[2020-11-25T16:51:09-0800] [MainThread] [I] [toil.leader] Issued job 'run_car_t_validity_assessment' kind-run_car_t_validity_assessment/instance-uyx66yka with job batch system ID: 37 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-25T16:51:09-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker assess_mhc_genes file:/scratch/drkthomp/jobStore kind-assess_mhc_genes/instance-qbajcs0c. +[2020-11-25T16:51:09-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c +[2020-11-25T16:51:09-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c +[2020-11-25T16:51:09-0800] [MainThread] [W] [toil.leader] Log from job kind-assess_mhc_genes/instance-qbajcs0c follows: +=========> + [2020-11-25T16:51:08-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-25T16:51:08-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/addons/assess_mhc_pathway.py", line 122, in assess_mhc_genes + ensgName = background_df.ix[ensg, 'Name'] + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/pandas/core/generic.py", line 5139, in __getattr__ + return object.__getattribute__(self, name) + AttributeError: 'DataFrame' object has no attribute 'ix' + [2020-11-25T16:51:09-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-25T16:51:09-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c with ID kind-assess_mhc_genes/instance-qbajcs0c to 1 +[2020-11-25T16:51:09-0800] [MainThread] [I] [toil.leader] Issued job 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c with job batch system ID: 38 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-25T16:51:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-25-2020 16:51:09: Exporting files/for-job/kind-assess_itx_resistance/instance-a2s3kbby/file-5o6f6lxt/immunotherapy_resistance_report.txt to output location +[2020-11-25T16:51:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-25-2020 16:51:09: Ran create immunotherapy resistance report on TEST successfully +[2020-11-25T16:51:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-25-2020 16:51:09: Exporting files/for-job/kind-assess_car_t_validity/instance-ijje8btr/file-kqn18q_a/car_t_target_report.txt to output location +[2020-11-25T16:51:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-25-2020 16:51:09: Ran car t validity assessment on TEST successfully +[2020-11-25T16:51:10-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpks10vl7n/worker_log.txt +[2020-11-25T16:51:10-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphklnu4mz/worker_log.txt +[2020-11-25T16:51:10-0800] [MainThread] [I] [toil.leader] Job ended: 'run_car_t_validity_assessment' kind-run_car_t_validity_assessment/instance-uyx66yka +[2020-11-25T16:51:10-0800] [MainThread] [I] [toil.leader] Job ended: 'run_itx_resistance_assessment' kind-run_itx_resistance_assessment/instance-00iiueib +[2020-11-25T16:51:10-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp75qerl0k/worker_log.txt +[2020-11-25T16:51:11-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker assess_mhc_genes file:/scratch/drkthomp/jobStore kind-assess_mhc_genes/instance-qbajcs0c. +[2020-11-25T16:51:11-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c +[2020-11-25T16:51:11-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c +[2020-11-25T16:51:11-0800] [MainThread] [W] [toil.leader] Log from job kind-assess_mhc_genes/instance-qbajcs0c follows: +=========> + [2020-11-25T16:51:10-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-25T16:51:10-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/addons/assess_mhc_pathway.py", line 122, in assess_mhc_genes + ensgName = background_df.ix[ensg, 'Name'] + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/pandas/core/generic.py", line 5139, in __getattr__ + return object.__getattribute__(self, name) + AttributeError: 'DataFrame' object has no attribute 'ix' + [2020-11-25T16:51:11-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-25T16:51:11-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c with ID kind-assess_mhc_genes/instance-qbajcs0c to 0 +[2020-11-25T16:51:11-0800] [MainThread] [W] [toil.leader] Job 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c with ID kind-assess_mhc_genes/instance-qbajcs0c is completely failed +[2020-11-25T17:10:15-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T18:10:16-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T19:10:16-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run diff --git a/errors/2020-11-25-20-22-32.txt b/errors/2020-11-25-20-22-32.txt new file mode 100644 index 00000000..a4e1d6c3 --- /dev/null +++ b/errors/2020-11-25-20-22-32.txt @@ -0,0 +1,17 @@ +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/bin/ProTECT", line 11, in + load_entry_point('protect', 'console_scripts', 'ProTECT')() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/pkg_resources/__init__.py", line 565, in load_entry_point + return get_distribution(dist).load_entry_point(group, name) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/pkg_resources/__init__.py", line 2631, in load_entry_point + return ep.load() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/pkg_resources/__init__.py", line 2291, in load + return self.resolve() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/pkg_resources/__init__.py", line 2297, in resolve + module = __import__(self.module_name, fromlist=['__name__'], level=0) + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 31, in + from protect.addons.assess_mhc_pathway import run_mhc_gene_assessment + File "/private/home/drkthomp/protect3/src/protect/addons/assess_mhc_pathway.py", line 126 + engsName = background_df.loc[engs, 'Name'] + ^ +TabError: inconsistent use of tabs and spaces in indentation diff --git a/errors/2020-11-25-20-23-15.txt b/errors/2020-11-25-20-23-15.txt new file mode 100644 index 00000000..d20fd705 --- /dev/null +++ b/errors/2020-11-25-20-23-15.txt @@ -0,0 +1,267 @@ +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xbtj2suv +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-uuibculn +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-yb2yf65u +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-aux6_f19 +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-2dtgopje +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-jkvqxjgf +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-06rjxk27 +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-q4vn5u6_ +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-0xdl6glf +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ge6vacd7 +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r7m8gybn +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-assess_mhc_genes/instance-qbajcs0c +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-eg7xdcao +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-7_6wjfvk +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-3nats6mz +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-d9qjg3jl +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-bv9m7hf3 +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-9uka9hrw +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-mppqh5vo +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ovr5_kcb +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_fusion/instance-g9_082sr +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-rd57kqhd +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xz7ir9sh +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-Job/instance-yfabv7w5 +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-k3tx4a2p +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ewrzdiri +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-2u48n_5f +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r1uv5wsv +[2020-11-25T20:23:18-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-v628y3w0 +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-uuibculn with job batch system ID: 0 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-aux6_f19 with job batch system ID: 1 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-d9qjg3jl with job batch system ID: 2 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-eg7xdcao with job batch system ID: 3 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mppqh5vo with job batch system ID: 4 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yb2yf65u with job batch system ID: 5 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-q4vn5u6_ with job batch system ID: 6 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ge6vacd7 with job batch system ID: 7 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-7_6wjfvk with job batch system ID: 8 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2u48n_5f with job batch system ID: 9 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jkvqxjgf with job batch system ID: 10 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bv9m7hf3 with job batch system ID: 11 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r1uv5wsv with job batch system ID: 12 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9uka9hrw with job batch system ID: 13 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ewrzdiri with job batch system ID: 14 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ovr5_kcb with job batch system ID: 15 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xbtj2suv with job batch system ID: 16 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0xdl6glf with job batch system ID: 17 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-v628y3w0 with job batch system ID: 18 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-k3tx4a2p with job batch system ID: 19 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-06rjxk27 with job batch system ID: 20 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-rd57kqhd with job batch system ID: 21 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xz7ir9sh with job batch system ID: 22 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r7m8gybn with job batch system ID: 23 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2dtgopje with job batch system ID: 24 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'reformat_star_fusion_output' kind-run_fusion/instance-g9_082sr with job batch system ID: 25 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-25T20:23:18-0800] [MainThread] [I] [toil.leader] Issued job 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c with job batch system ID: 26 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpy8eqr_i0/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmprg_ppksy/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3joys_i0/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp1copbw06/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmprk6xe8lf/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmplxw6c6mk/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmptuikdwrd/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpz2_3x3eh/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfll2ec5g/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpbh8iznj4/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpt1xxcind/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpin6ia45z/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_firibxf/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfgruwusx/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpc574ytw0/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_tn6utui/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp4y923n6h/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjk47elb7/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_tk50uxl/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp110iirqg/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpyj529gwd/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpp_6x1qsn/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpb_k6hos2/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpmy1remj3/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphqu3h62b/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpy4omwrfi/worker_log.txt +[2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpjqsxu_fl/worker_log.txt +[2020-11-25T20:23:20-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker reformat_star_fusion_output file:/scratch/drkthomp/jobStore kind-run_fusion/instance-g9_082sr. +[2020-11-25T20:23:20-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'reformat_star_fusion_output' kind-run_fusion/instance-g9_082sr +[2020-11-25T20:23:20-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'reformat_star_fusion_output' kind-run_fusion/instance-g9_082sr +[2020-11-25T20:23:20-0800] [MainThread] [W] [toil.leader] Log from job kind-run_fusion/instance-g9_082sr follows: +=========> + [2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-25T20:23:19-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/fusion.py", line 407, in reformat_star_fusion_output + transcripts) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/fusion.py", line 300, in split_fusion_transcript + trans = string.maketrans(forward, reverse) + AttributeError: module 'string' has no attribute 'maketrans' + [2020-11-25T20:23:20-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-25T20:23:20-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'reformat_star_fusion_output' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr to 1 +[2020-11-25T20:23:20-0800] [MainThread] [I] [toil.leader] 26 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T20:23:20-0800] [MainThread] [I] [toil.leader] Issued job 'reformat_star_fusion_output' kind-run_fusion/instance-g9_082sr with job batch system ID: 27 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-25T20:23:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmphlgk1pqp/worker_log.txt +[2020-11-25T20:23:20-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker assess_mhc_genes file:/scratch/drkthomp/jobStore kind-assess_mhc_genes/instance-qbajcs0c. +[2020-11-25T20:23:20-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c +[2020-11-25T20:23:20-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c +[2020-11-25T20:23:20-0800] [MainThread] [W] [toil.leader] Log from job kind-assess_mhc_genes/instance-qbajcs0c follows: +=========> + [2020-11-25T20:23:19-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-25T20:23:19-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/addons/assess_mhc_pathway.py", line 126, in assess_mhc_genes + engsName = background_df.loc[engs, 'Name'] + NameError: name 'engs' is not defined + [2020-11-25T20:23:20-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + Adipose Tissue Adrenal Gland ... Uterus Vagina + ENSG00000001167 9.441831 7.354315 ... 11.294517 9.610609 + ENSG00000019582 1308.079388 828.570087 ... 638.652456 515.248823 + ENSG00000044574 140.313642 163.581842 ... 125.750989 105.357563 + ENSG00000064490 51.472339 37.139550 ... 73.725525 61.068943 + ENSG00000066136 33.389298 29.199442 ... 53.899833 44.385150 + ENSG00000080824 321.366496 429.761806 ... 367.704926 338.044289 + ENSG00000092010 155.638792 203.401612 ... 192.667241 171.754740 + ENSG00000096384 458.153226 633.832074 ... 669.413042 528.507774 + ENSG00000100600 89.954110 41.504392 ... 41.405598 61.034524 + ENSG00000100911 99.632905 224.925473 ... 139.854085 120.661883 + ENSG00000109971 407.684921 596.504378 ... 406.804435 376.050226 + ENSG00000111537 0.364658 0.019921 ... 0.136539 0.134881 + ENSG00000118260 11.548167 6.888884 ... 11.627579 10.795009 + ENSG00000120837 34.698551 43.729688 ... 40.190208 27.694173 + ENSG00000126803 8.767770 5.575119 ... 36.132302 15.706558 + ENSG00000127022 133.042096 131.381548 ... 183.315955 132.958215 + ENSG00000131467 40.193737 38.903497 ... 37.103956 38.081321 + ENSG00000133111 3.988484 2.208118 ... 7.120134 4.077026 + ENSG00000135047 183.588838 85.690312 ... 128.938812 59.014368 + ENSG00000143390 15.580966 26.094969 ... 27.950625 15.205987 + ENSG00000163131 49.576132 20.159561 ... 7.039600 14.848325 + ENSG00000164733 476.621914 322.742425 ... 386.919329 572.434397 + ENSG00000166710 5382.638402 3521.849670 ... 3809.488060 3394.361021 + ENSG00000167004 159.788698 169.695816 ... 201.892229 173.300456 + ENSG00000168394 34.962744 29.307153 ... 38.214449 34.614391 + ENSG00000170606 37.530706 53.073652 ... 38.496652 45.001358 + ENSG00000173110 15.473577 9.341471 ... 8.496765 12.310593 + ENSG00000179218 247.208373 263.116842 ... 296.774581 272.151740 + ENSG00000179583 9.280137 4.136846 ... 6.634200 5.378919 + ENSG00000204257 92.023736 72.992062 ... 43.549149 35.527560 + ENSG00000204287 412.033988 257.014398 ... 165.511750 147.917202 + ENSG00000204388 270.308462 246.780903 ... 543.674118 495.294052 + ENSG00000204389 359.267238 239.192157 ... 610.185599 568.740254 + ENSG00000204390 1.051380 1.346538 ... 2.497306 1.291429 + ENSG00000216490 115.825840 120.551779 ... 18.183466 23.738014 + ENSG00000231925 44.824282 63.769936 ... 49.724079 46.997271 + ENSG00000232810 0.720197 0.490315 ... 0.179615 0.140714 + ENSG00000242574 41.206179 36.804025 ... 19.483655 15.779626 + ENSG00000250264 0.705807 0.752759 ... 1.019871 0.983095 + + [39 rows x 57 columns] +<========= +[2020-11-25T20:23:20-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c with ID kind-assess_mhc_genes/instance-qbajcs0c to 1 +[2020-11-25T20:23:20-0800] [MainThread] [I] [toil.leader] Issued job 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c with job batch system ID: 28 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-25T20:23:21-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker reformat_star_fusion_output file:/scratch/drkthomp/jobStore kind-run_fusion/instance-g9_082sr. +[2020-11-25T20:23:21-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'reformat_star_fusion_output' kind-run_fusion/instance-g9_082sr +[2020-11-25T20:23:21-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'reformat_star_fusion_output' kind-run_fusion/instance-g9_082sr +[2020-11-25T20:23:21-0800] [MainThread] [W] [toil.leader] Log from job kind-run_fusion/instance-g9_082sr follows: +=========> + [2020-11-25T20:23:20-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-25T20:23:20-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/fusion.py", line 407, in reformat_star_fusion_output + transcripts) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/fusion.py", line 300, in split_fusion_transcript + trans = string.maketrans(forward, reverse) + AttributeError: module 'string' has no attribute 'maketrans' + [2020-11-25T20:23:21-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-25T20:23:21-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'reformat_star_fusion_output' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr to 0 +[2020-11-25T20:23:21-0800] [MainThread] [W] [toil.leader] Job 'reformat_star_fusion_output' kind-run_fusion/instance-g9_082sr with ID kind-run_fusion/instance-g9_082sr is completely failed +[2020-11-25T20:23:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmppsadoco7/worker_log.txt +[2020-11-25T20:23:22-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker assess_mhc_genes file:/scratch/drkthomp/jobStore kind-assess_mhc_genes/instance-qbajcs0c. +[2020-11-25T20:23:22-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c +[2020-11-25T20:23:22-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c +[2020-11-25T20:23:22-0800] [MainThread] [W] [toil.leader] Log from job kind-assess_mhc_genes/instance-qbajcs0c follows: +=========> + [2020-11-25T20:23:21-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-25T20:23:21-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/addons/assess_mhc_pathway.py", line 126, in assess_mhc_genes + engsName = background_df.loc[engs, 'Name'] + NameError: name 'engs' is not defined + [2020-11-25T20:23:22-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + Adipose Tissue Adrenal Gland ... Uterus Vagina + ENSG00000001167 9.441831 7.354315 ... 11.294517 9.610609 + ENSG00000019582 1308.079388 828.570087 ... 638.652456 515.248823 + ENSG00000044574 140.313642 163.581842 ... 125.750989 105.357563 + ENSG00000064490 51.472339 37.139550 ... 73.725525 61.068943 + ENSG00000066136 33.389298 29.199442 ... 53.899833 44.385150 + ENSG00000080824 321.366496 429.761806 ... 367.704926 338.044289 + ENSG00000092010 155.638792 203.401612 ... 192.667241 171.754740 + ENSG00000096384 458.153226 633.832074 ... 669.413042 528.507774 + ENSG00000100600 89.954110 41.504392 ... 41.405598 61.034524 + ENSG00000100911 99.632905 224.925473 ... 139.854085 120.661883 + ENSG00000109971 407.684921 596.504378 ... 406.804435 376.050226 + ENSG00000111537 0.364658 0.019921 ... 0.136539 0.134881 + ENSG00000118260 11.548167 6.888884 ... 11.627579 10.795009 + ENSG00000120837 34.698551 43.729688 ... 40.190208 27.694173 + ENSG00000126803 8.767770 5.575119 ... 36.132302 15.706558 + ENSG00000127022 133.042096 131.381548 ... 183.315955 132.958215 + ENSG00000131467 40.193737 38.903497 ... 37.103956 38.081321 + ENSG00000133111 3.988484 2.208118 ... 7.120134 4.077026 + ENSG00000135047 183.588838 85.690312 ... 128.938812 59.014368 + ENSG00000143390 15.580966 26.094969 ... 27.950625 15.205987 + ENSG00000163131 49.576132 20.159561 ... 7.039600 14.848325 + ENSG00000164733 476.621914 322.742425 ... 386.919329 572.434397 + ENSG00000166710 5382.638402 3521.849670 ... 3809.488060 3394.361021 + ENSG00000167004 159.788698 169.695816 ... 201.892229 173.300456 + ENSG00000168394 34.962744 29.307153 ... 38.214449 34.614391 + ENSG00000170606 37.530706 53.073652 ... 38.496652 45.001358 + ENSG00000173110 15.473577 9.341471 ... 8.496765 12.310593 + ENSG00000179218 247.208373 263.116842 ... 296.774581 272.151740 + ENSG00000179583 9.280137 4.136846 ... 6.634200 5.378919 + ENSG00000204257 92.023736 72.992062 ... 43.549149 35.527560 + ENSG00000204287 412.033988 257.014398 ... 165.511750 147.917202 + ENSG00000204388 270.308462 246.780903 ... 543.674118 495.294052 + ENSG00000204389 359.267238 239.192157 ... 610.185599 568.740254 + ENSG00000204390 1.051380 1.346538 ... 2.497306 1.291429 + ENSG00000216490 115.825840 120.551779 ... 18.183466 23.738014 + ENSG00000231925 44.824282 63.769936 ... 49.724079 46.997271 + ENSG00000232810 0.720197 0.490315 ... 0.179615 0.140714 + ENSG00000242574 41.206179 36.804025 ... 19.483655 15.779626 + ENSG00000250264 0.705807 0.752759 ... 1.019871 0.983095 + + [39 rows x 57 columns] +<========= +[2020-11-25T20:23:22-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c with ID kind-assess_mhc_genes/instance-qbajcs0c to 0 +[2020-11-25T20:23:22-0800] [MainThread] [W] [toil.leader] Job 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c with ID kind-assess_mhc_genes/instance-qbajcs0c is completely failed diff --git a/errors/2020-11-25-20-26-48.txt b/errors/2020-11-25-20-26-48.txt new file mode 100644 index 00000000..88611493 --- /dev/null +++ b/errors/2020-11-25-20-26-48.txt @@ -0,0 +1,105 @@ +[2020-11-25T20:26:50-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-assess_mhc_genes/instance-qbajcs0c +[2020-11-25T20:26:50-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_fusion/instance-g9_082sr +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-uuibculn with job batch system ID: 0 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-aux6_f19 with job batch system ID: 1 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-d9qjg3jl with job batch system ID: 2 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-eg7xdcao with job batch system ID: 3 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mppqh5vo with job batch system ID: 4 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-yb2yf65u with job batch system ID: 5 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-q4vn5u6_ with job batch system ID: 6 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ge6vacd7 with job batch system ID: 7 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-7_6wjfvk with job batch system ID: 8 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2u48n_5f with job batch system ID: 9 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jkvqxjgf with job batch system ID: 10 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bv9m7hf3 with job batch system ID: 11 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r1uv5wsv with job batch system ID: 12 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9uka9hrw with job batch system ID: 13 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ewrzdiri with job batch system ID: 14 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ovr5_kcb with job batch system ID: 15 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xbtj2suv with job batch system ID: 16 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-0xdl6glf with job batch system ID: 17 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-v628y3w0 with job batch system ID: 18 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-k3tx4a2p with job batch system ID: 19 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-06rjxk27 with job batch system ID: 20 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-rd57kqhd with job batch system ID: 21 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xz7ir9sh with job batch system ID: 22 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r7m8gybn with job batch system ID: 23 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-2dtgopje with job batch system ID: 24 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'reformat_star_fusion_output' kind-run_fusion/instance-g9_082sr with job batch system ID: 25 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-25T20:26:50-0800] [MainThread] [I] [toil.leader] Issued job 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c with job batch system ID: 26 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpcroxzr5g/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmplis8o20z/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp1pwmpxm2/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmppr3ixc5q/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp3jrtxdrk/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpd78na29b/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmprvnofqin/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp2avyprz8/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp_cgkt1dw/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpp4bt0uwq/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmplfb62g8q/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpp92fdip9/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmph53lq5jh/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfxa0gi97/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpwuj9o1_k/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp1ixjzbyt/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0awqc1i8/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpfj97spsb/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpifnlwwpb/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpxu1s5cgk/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp1hdr82am/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpsbvbwuzr/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp5730lajq/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp0o9fmpmr/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpypkfdii5/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpdzdsa5gg/worker_log.txt +[2020-11-25T20:26:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpaziw60m0/worker_log.txt +[2020-11-25T20:26:52-0800] [MainThread] [I] [toil.leader] 27 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T20:26:52-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-25-2020 20:26:52: Exporting files/for-job/kind-run_fusion/instance-g9_082sr/file-c9dhzi51/fusion_results.bedpe to output location +[2020-11-25T20:26:52-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-25-2020 20:26:52: Reformatted STAR-Fusion output for TEST successfully +[2020-11-25T20:26:52-0800] [MainThread] [I] [toil.leader] Job ended: 'reformat_star_fusion_output' kind-run_fusion/instance-g9_082sr +[2020-11-25T20:26:52-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-906qbyom with job batch system ID: 27 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-25T20:26:53-0800] [MainThread] [I] [toil.leader] Job ended: 'assess_mhc_genes' kind-assess_mhc_genes/instance-qbajcs0c +[2020-11-25T20:26:53-0800] [MainThread] [I] [toil.leader] Issued job 'run_mhc_gene_assessment' kind-run_mhc_gene_assessment/instance-qkfz94bs with job batch system ID: 28 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-25T20:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp6tp2rgb1/worker_log.txt +[2020-11-25T20:26:53-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-25-2020 20:26:53: Exporting files/for-job/kind-assess_mhc_genes/instance-qbajcs0c/file-qypnyd73/mhc_pathway_report.txt to output location +[2020-11-25T20:26:53-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-25-2020 20:26:53: Ran mhc gene assessment on TEST successfully +[2020-11-25T20:26:53-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-906qbyom +[2020-11-25T20:26:53-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-j5sf3v2f with job batch system ID: 29 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-25T20:26:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpxf6cdfoy/worker_log.txt +[2020-11-25T20:26:53-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mhc_gene_assessment' kind-run_mhc_gene_assessment/instance-qkfz94bs +[2020-11-25T20:26:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpoxdyuwtx/worker_log.txt +[2020-11-25T20:26:54-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-j5sf3v2f +[2020-11-25T20:26:54-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-846388av with job batch system ID: 30 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-25T20:26:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmp8zrdaqt4/worker_log.txt +[2020-11-25T20:26:55-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-846388av +[2020-11-25T20:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'delete_fastqs' kind-delete_fastqs/instance-acifqje4 with job batch system ID: 31 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-25T20:26:55-0800] [MainThread] [I] [toil.leader] Issued job 'delete_fastqs' kind-delete_fastqs/instance-av8_0fvi with job batch system ID: 32 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-25T20:26:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpo4p6bsik/worker_log.txt +[2020-11-25T20:26:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-0afffc36-5183-48a3-b8b6-f3fc5f1d5a9c-e8611b53-bdab-4598-acae-87977f99831e/tmpi_pmmlbc/worker_log.txt +[2020-11-25T20:26:56-0800] [MainThread] [I] [toil.leader] Job ended: 'delete_fastqs' kind-delete_fastqs/instance-av8_0fvi +[2020-11-25T20:26:56-0800] [MainThread] [I] [toil.leader] Job ended: 'delete_fastqs' kind-delete_fastqs/instance-acifqje4 +[2020-11-25T20:26:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-25-2020 20:26:57: Deleting "TEST:normal_dna_fastq_2" from the filestore. +[2020-11-25T20:26:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-25-2020 20:26:57: Deleting "TEST:normal_dna_fastq_1" from the filestore. +[2020-11-25T20:26:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-25-2020 20:26:57: Deleting "TEST:tumor_dna_fastq_1" from the filestore. +[2020-11-25T20:26:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-25-2020 20:26:57: Deleting "TEST:tumor_rna_fastq_2" from the filestore. +[2020-11-25T20:26:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-25-2020 20:26:57: Deleting "TEST:tumor_rna_fastq_1" from the filestore. +[2020-11-25T20:26:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-25-2020 20:26:57: Deleting "TEST:tumor_dna_fastq_2" from the filestore. +[2020-11-25T21:26:53-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T22:26:54-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-25T23:26:54-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T00:26:55-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T01:26:56-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T02:26:56-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T03:26:57-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T04:26:58-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T05:26:58-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T06:26:59-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T07:27:00-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T08:27:01-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T09:27:01-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T10:27:02-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T11:27:03-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T12:27:03-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T13:27:04-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run diff --git a/errors/2020-11-26-14-11-04.txt b/errors/2020-11-26-14-11-04.txt new file mode 100644 index 00000000..e1ca13c8 --- /dev/null +++ b/errors/2020-11-26-14-11-04.txt @@ -0,0 +1,1989 @@ +[2020-11-26T14:11:06-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-26T14:11:06-0800] [MainThread] [I] [toil.leader] Issued job 'parse_config_file' kind-parse_config_file/instance-2sb4l2pw with job batch system ID: 0 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp2487etrj/worker_log.txt +[2020-11-26T14:11:08-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:08: Parsing config file +[2020-11-26T14:11:08-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:08: Obtaining tool inputs +[2020-11-26T14:11:08-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:08: Obtained tool inputs +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Job ended: 'parse_config_file' kind-parse_config_file/instance-2sb4l2pw +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] 0 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-rt27g43c with job batch system ID: 1 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-252h7pe2 with job batch system ID: 2 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-p6sijzua with job batch system ID: 3 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-xu2ab0ek with job batch system ID: 4 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-blt72gql with job batch system ID: 5 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1i_438ao with job batch system ID: 6 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-uncc2uyp with job batch system ID: 7 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-zo8guom3 with job batch system ID: 8 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-z_s6a4vi with job batch system ID: 9 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-8lyxphjw with job batch system ID: 10 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-stvvfnfc with job batch system ID: 11 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-yx8uw8_4 with job batch system ID: 12 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-2ggg4dy5 with job batch system ID: 13 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-g56av4ma with job batch system ID: 14 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-l5qbfjmz with job batch system ID: 15 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-8d8waic9 with job batch system ID: 16 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-wkp3huo6 with job batch system ID: 17 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6aec6cy6 with job batch system ID: 18 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-grc2e7ip with job batch system ID: 19 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-vc14q6xq with job batch system ID: 20 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-y10uqtl6 with job batch system ID: 21 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ktejomue with job batch system ID: 22 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-b4tp4gon with job batch system ID: 23 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-hnq1yayl with job batch system ID: 24 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-kxg9_jxk with job batch system ID: 25 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-p7u1ptj7 with job batch system ID: 26 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-gpuqnh47 with job batch system ID: 27 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ekqicjql with job batch system ID: 28 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:08-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-8tpedbaf with job batch system ID: 29 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpscj057ei/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpnwqns9x5/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpocutoqpz/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpnujg5e8o/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp8yars0qk/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp1kyseema/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp21hn9vet/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmppjgwmfmc/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpoix6ltz3/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpfe_tx7o6/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpm9jg8dpp/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpuulmuf9c/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp8qdsj537/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmple1w11ba/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmphthn3chg/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpih2k651n/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpmwskxyir/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpfzd2_h_g/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpwkk42nx3/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpswpxq1lf/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpovewaybi/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpbiidww6i/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpzdt53dln/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpor8_ipir/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpi86lw7_o/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpyabyds4j/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp55fzqe6f/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpp_xzabvo/worker_log.txt +[2020-11-26T14:11:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp2y7higft/worker_log.txt +[2020-11-26T14:11:10-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:10: Obtaining file (indexes:genome_fai) to the file job store +[2020-11-26T14:11:10-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-blt72gql +[2020-11-26T14:11:10-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-1i_438ao +[2020-11-26T14:11:10-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-wkp3huo6 +[2020-11-26T14:11:10-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-zo8guom3 +[2020-11-26T14:11:10-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:10: Obtaining file (indexes:genome_dict) to the file job store +[2020-11-26T14:11:10-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:10: Obtaining file (strelka:config_file) to the file job store +[2020-11-26T14:11:10-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:10: Obtaining file (indexes:cosmic_idx) to the file job store +[2020-11-26T14:11:10-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:10: Obtaining file (radia:retrogene_beds) to the file job store +[2020-11-26T14:11:10-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-g56av4ma +[2020-11-26T14:11:10-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-stvvfnfc +[2020-11-26T14:11:10-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-8d8waic9 +[2020-11-26T14:11:10-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-p7u1ptj7 +[2020-11-26T14:11:10-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-hnq1yayl +[2020-11-26T14:11:10-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-gpuqnh47 +[2020-11-26T14:11:10-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-kxg9_jxk +[2020-11-26T14:11:10-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ekqicjql +[2020-11-26T14:11:10-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-8tpedbaf +[2020-11-26T14:11:10-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-l5qbfjmz +[2020-11-26T14:11:11-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:11: Obtaining file (indexes:dbsnp_tbi) to the file job store +[2020-11-26T14:11:11-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:11: Obtaining file (radia:gencode_beds) to the file job store +[2020-11-26T14:11:11-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:11: Obtaining file (reports:itx_resistance_file) to the file job store +[2020-11-26T14:11:11-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:11: Obtaining file (reports:immune_resistance_pathways_file) to the file job store +[2020-11-26T14:11:11-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:11: Obtaining file (mhci:method_file) to the file job store +[2020-11-26T14:11:11-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:11: Obtaining file (reports:mhc_pathways_file) to the file job store +[2020-11-26T14:11:11-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:11: Obtaining file (reports:car_t_targets_file) to the file job store +[2020-11-26T14:11:11-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:11: Obtaining file (mhcii:method_file) to the file job store +[2020-11-26T14:11:11-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:11: Obtaining file (radia:pseudogene_beds) to the file job store +[2020-11-26T14:11:11-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-8lyxphjw +[2020-11-26T14:11:11-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:11: Obtaining file (indexes:dbsnp_idx) to the file job store +[2020-11-26T14:11:11-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:11: Obtaining file (transgene:gencode_peptide_fasta) to the file job store +[2020-11-26T14:11:11-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-grc2e7ip +[2020-11-26T14:11:12-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:12: Obtaining file (indexes:cosmic_vcf) to the file job store +[2020-11-26T14:11:12-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-uncc2uyp +[2020-11-26T14:11:12-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-yx8uw8_4 +[2020-11-26T14:11:13-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:13: Obtaining file (radia:cosmic_beds) to the file job store +[2020-11-26T14:11:13-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-y10uqtl6 +[2020-11-26T14:11:13-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-vc14q6xq +[2020-11-26T14:11:13-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:13: Obtaining file (transgene:gencode_annotation_gtf) to the file job store +[2020-11-26T14:11:13-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:13: Obtaining file (transgene:gencode_transcript_fasta) to the file job store +[2020-11-26T14:11:41-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ktejomue +[2020-11-26T14:11:41-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-xu2ab0ek +[2020-11-26T14:11:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:42: Obtaining file (transgene:genome_fasta) to the file job store +[2020-11-26T14:11:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:42: Obtaining file (indexes:genome_fasta) to the file job store +[2020-11-26T14:11:44-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:11:44: Obtaining file (radia:dbsnp_beds) to the file job store +[2020-11-26T14:11:44-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-2ggg4dy5 +[2020-11-26T14:12:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:12:00: Obtaining file (rsem:index) to the file job store +[2020-11-26T14:12:00-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-p6sijzua +[2020-11-26T14:12:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:12:00: Obtaining file (snpeff:index) to the file job store +[2020-11-26T14:12:00-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-6aec6cy6 +[2020-11-26T14:12:24-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:12:24: Obtaining file (indexes:dbsnp_vcf) to the file job store +[2020-11-26T14:12:24-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-z_s6a4vi +[2020-11-26T14:12:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:12:54: Obtaining file (bwa:index) to the file job store +[2020-11-26T14:12:54-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-252h7pe2 +[2020-11-26T14:13:08-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:13:08: Obtaining file (phlat:index) to the file job store +[2020-11-26T14:13:08-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-b4tp4gon +[2020-11-26T14:14:55-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-rt27g43c +[2020-11-26T14:14:55-0800] [MainThread] [I] [toil.leader] Issued job 'launch_protect' kind-launch_protect/instance-0om1l7xc with job batch system ID: 30 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:14:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:14:55: Obtaining file (star:index) to the file job store +[2020-11-26T14:14:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpey9gta1m/worker_log.txt +[2020-11-26T14:14:57-0800] [MainThread] [I] [toil.leader] Job ended: 'launch_protect' kind-launch_protect/instance-0om1l7xc +[2020-11-26T14:14:57-0800] [MainThread] [I] [toil.leader] Issued job 'prepare_samples' kind-prepare_samples/instance-sicwqye7 with job batch system ID: 31 and cores: 1, disk: 40.0 G, and memory: 2.0 G +[2020-11-26T14:14:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmppjgv1j05/worker_log.txt +[2020-11-26T14:15:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:15:15: Downloading Inputs for TEST +[2020-11-26T14:15:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:15:15: Obtaining file (TEST:tumor_rna_fastq_1) to the file job store +[2020-11-26T14:15:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:15:15: Obtaining file (TEST:tumor_dna_fastq_2) to the file job store +[2020-11-26T14:15:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:15:15: Obtaining file (TEST:tumor_dna_fastq_1) to the file job store +[2020-11-26T14:15:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:15:15: Obtaining file (TEST:tumor_rna_fastq_2) to the file job store +[2020-11-26T14:15:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:15:15: Obtaining file (TEST:normal_dna_fastq_1) to the file job store +[2020-11-26T14:15:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:15:15: Obtaining file (TEST:normal_dna_fastq_2) to the file job store +[2020-11-26T14:15:15-0800] [MainThread] [I] [toil.leader] Job ended: 'prepare_samples' kind-prepare_samples/instance-sicwqye7 +[2020-11-26T14:15:15-0800] [MainThread] [I] [toil.leader] Issued job 'get_patient_fastqs' kind-get_patient_fastqs/instance-hn86u108 with job batch system ID: 32 and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-11-26T14:15:15-0800] [MainThread] [I] [toil.leader] Issued job 'get_patient_fastqs' kind-get_patient_fastqs/instance-z9ye1mcg with job batch system ID: 33 and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-11-26T14:15:15-0800] [MainThread] [I] [toil.leader] Issued job 'get_patient_fastqs' kind-get_patient_fastqs/instance-qy6e6lbt with job batch system ID: 34 and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-11-26T14:15:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp74hyed18/worker_log.txt +[2020-11-26T14:15:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpco7olhxg/worker_log.txt +[2020-11-26T14:15:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp3_c3gs2a/worker_log.txt +[2020-11-26T14:15:17-0800] [MainThread] [I] [toil.leader] Job ended: 'get_patient_fastqs' kind-get_patient_fastqs/instance-hn86u108 +[2020-11-26T14:15:17-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-fpxwmnm0 with job batch system ID: 35 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:15:17-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-1v9rwibu with job batch system ID: 36 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-26T14:15:17-0800] [MainThread] [I] [toil.leader] Job ended: 'get_patient_fastqs' kind-get_patient_fastqs/instance-z9ye1mcg +[2020-11-26T14:15:17-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-7436i_1u with job batch system ID: 37 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:15:17-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-ulj3tq4p with job batch system ID: 38 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-26T14:15:17-0800] [MainThread] [I] [toil.leader] Job ended: 'get_patient_fastqs' kind-get_patient_fastqs/instance-qy6e6lbt +[2020-11-26T14:15:17-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-y7p631p0 with job batch system ID: 39 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:15:17-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-8cxak175 with job batch system ID: 40 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:15:17-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpatldfi0f/worker_log.txt +[2020-11-26T14:15:17-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpa8p8v35w/worker_log.txt +[2020-11-26T14:15:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpxg0u36q8/worker_log.txt +[2020-11-26T14:15:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp1cuj3f_k/worker_log.txt +[2020-11-26T14:15:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpdc0uasf3/worker_log.txt +[2020-11-26T14:15:18-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-fpxwmnm0 +[2020-11-26T14:15:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-q2libskg with job batch system ID: 41 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-26T14:15:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp4dcbitt9/worker_log.txt +[2020-11-26T14:15:18-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-7436i_1u +[2020-11-26T14:15:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-8v93d6hk with job batch system ID: 42 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-26T14:15:18-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-y7p631p0 +[2020-11-26T14:15:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-q4kajmqf with job batch system ID: 43 and cores: 80, disk: 8.2 G, and memory: 2.0 G +[2020-11-26T14:15:18-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-8cxak175 +[2020-11-26T14:15:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_cutadapt' kind-run_cutadapt/instance-zeqlnbw4 with job batch system ID: 44 and cores: 1, disk: 1.8 G, and memory: 2.0 G +[2020-11-26T14:15:18-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-1v9rwibu +[2020-11-26T14:15:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-vqkkid9v with job batch system ID: 45 and cores: 80, disk: 13.4 G, and memory: 12.0 G +[2020-11-26T14:15:18-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-ulj3tq4p +[2020-11-26T14:15:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-kce5wdcp with job batch system ID: 46 and cores: 80, disk: 12.8 G, and memory: 12.0 G +[2020-11-26T14:15:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp1b7uy467/worker_log.txt +[2020-11-26T14:15:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpk630e118/worker_log.txt +[2020-11-26T14:16:10-0800] [MainThread] [I] [toil.leader] Job ended: 'run_cutadapt' kind-run_cutadapt/instance-zeqlnbw4 +[2020-11-26T14:16:10-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-vswly9bj with job batch system ID: 47 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:16:11-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:16:11: Ran cutadapt on TEST successfully +[2020-11-26T14:16:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpethsum9g/worker_log.txt +[2020-11-26T14:22:04-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:22:04: Ran phlat on TEST:tumor_rna successfully +[2020-11-26T14:22:05-0800] [MainThread] [I] [toil.leader] Job ended: 'run_phlat' kind-run_phlat/instance-q4kajmqf +[2020-11-26T14:22:05-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-tqoy250f with job batch system ID: 48 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:22:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmptywb02cr/worker_log.txt +[2020-11-26T14:22:21-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:22:21: Ran phlat on TEST:tumor_dna successfully +[2020-11-26T14:22:21-0800] [MainThread] [I] [toil.leader] Job ended: 'run_phlat' kind-run_phlat/instance-q2libskg +[2020-11-26T14:22:21-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ttb3hy7r with job batch system ID: 49 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:22:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp5w6_bkov/worker_log.txt +[2020-11-26T14:22:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpikxiy_gp/worker_log.txt +[2020-11-26T14:22:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp83ag7soz/worker_log.txt +[2020-11-26T14:22:22-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ttb3hy7r +[2020-11-26T14:22:22-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-vswly9bj +[2020-11-26T14:22:22-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-9a1recg5 with job batch system ID: 50 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-26T14:22:22-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-tqoy250f +[2020-11-26T14:22:23-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmprn0nmyt2/worker_log.txt +[2020-11-26T14:25:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:25:01: Ran bwa on TEST:tumor_dna successfully +[2020-11-26T14:25:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_bwa' kind-run_bwa/instance-vqkkid9v +[2020-11-26T14:25:02-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-_3_wmlwl with job batch system ID: 51 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:25:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpynfnzrhe/worker_log.txt +[2020-11-26T14:25:12-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:25:12: Ran bwa on TEST:normal_dna successfully +[2020-11-26T14:25:13-0800] [MainThread] [I] [toil.leader] Job ended: 'run_bwa' kind-run_bwa/instance-kce5wdcp +[2020-11-26T14:25:13-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-yuscfm9u with job batch system ID: 52 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:25:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpc1j9twb3/worker_log.txt +[2020-11-26T14:25:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpjvj_qog_/worker_log.txt +[2020-11-26T14:25:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp5kypa_lw/worker_log.txt +[2020-11-26T14:25:14-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-_3_wmlwl +[2020-11-26T14:25:14-0800] [MainThread] [I] [toil.leader] Issued job 'bam_conversion' kind-bam_conversion/instance-t8fbj72h with job batch system ID: 53 and cores: 1, disk: 1.7 G, and memory: 12.0 G +[2020-11-26T14:25:14-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-yuscfm9u +[2020-11-26T14:25:14-0800] [MainThread] [I] [toil.leader] Issued job 'bam_conversion' kind-bam_conversion/instance-9rrtecju with job batch system ID: 54 and cores: 1, disk: 1.6 G, and memory: 12.0 G +[2020-11-26T14:25:14-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-9a1recg5 +[2020-11-26T14:25:14-0800] [MainThread] [I] [toil.leader] Issued job 'run_star' kind-run_star/instance-ccd4mgp2 with job batch system ID: 55 and cores: 80, disk: 53.7 G, and memory: 48.6 G +[2020-11-26T14:25:15-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp7oe2grcp/worker_log.txt +[2020-11-26T14:25:15-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpu0ad0dzm/worker_log.txt +[2020-11-26T14:26:22-0800] [MainThread] [I] [toil.leader] Job ended: 'bam_conversion' kind-bam_conversion/instance-9rrtecju +[2020-11-26T14:26:22-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-kgtbwyfd with job batch system ID: 56 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:26:22-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:26:22: Ran sam2bam on TEST:normal_dna successfully +[2020-11-26T14:26:23-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpsym3s9g1/worker_log.txt +[2020-11-26T14:26:23-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-kgtbwyfd +[2020-11-26T14:26:23-0800] [MainThread] [I] [toil.leader] Issued job 'fix_bam_header' kind-fix_bam_header/instance-1d79f70b with job batch system ID: 57 and cores: 1, disk: 1.6 G, and memory: 12.0 G +[2020-11-26T14:26:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmplv8enpq8/worker_log.txt +[2020-11-26T14:26:29-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:26:29: Ran reheader on TEST:normal_dna successfully +[2020-11-26T14:26:30-0800] [MainThread] [I] [toil.leader] Job ended: 'fix_bam_header' kind-fix_bam_header/instance-1d79f70b +[2020-11-26T14:26:30-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-dxink9kr with job batch system ID: 58 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:26:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpmt84ns9a/worker_log.txt +[2020-11-26T14:26:32-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-dxink9kr +[2020-11-26T14:26:32-0800] [MainThread] [I] [toil.leader] Issued job 'add_readgroups' kind-add_readgroups/instance-s0qda4k5 with job batch system ID: 59 and cores: 1, disk: 1.3 G, and memory: 12.0 G +[2020-11-26T14:26:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp49toxcj0/worker_log.txt +[2020-11-26T14:26:33-0800] [MainThread] [I] [toil.leader] Job ended: 'bam_conversion' kind-bam_conversion/instance-t8fbj72h +[2020-11-26T14:26:33-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-f4hwh0wg with job batch system ID: 60 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:26:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:26:33: Ran sam2bam on TEST:tumor_dna successfully +[2020-11-26T14:26:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpju1luf0y/worker_log.txt +[2020-11-26T14:26:35-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-f4hwh0wg +[2020-11-26T14:26:35-0800] [MainThread] [I] [toil.leader] Issued job 'fix_bam_header' kind-fix_bam_header/instance-1tirhcdz with job batch system ID: 61 and cores: 1, disk: 1.7 G, and memory: 12.0 G +[2020-11-26T14:26:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp5r7xnz2r/worker_log.txt +[2020-11-26T14:26:41-0800] [MainThread] [I] [toil.leader] Job ended: 'fix_bam_header' kind-fix_bam_header/instance-1tirhcdz +[2020-11-26T14:26:41-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-t810tc85 with job batch system ID: 62 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:26:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:26:41: Ran reheader on TEST:tumor_dna successfully +[2020-11-26T14:26:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmplnsuxvp0/worker_log.txt +[2020-11-26T14:26:42-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-t810tc85 +[2020-11-26T14:26:42-0800] [MainThread] [I] [toil.leader] Issued job 'add_readgroups' kind-add_readgroups/instance-5h5oumic with job batch system ID: 63 and cores: 1, disk: 1.5 G, and memory: 12.0 G +[2020-11-26T14:26:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmprpo97nam/worker_log.txt +[2020-11-26T14:28:11-0800] [MainThread] [I] [toil.leader] Job ended: 'add_readgroups' kind-add_readgroups/instance-s0qda4k5 +[2020-11-26T14:28:11-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-018i7hj0 with job batch system ID: 64 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:28:11-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:28:11: Ran add_read_groups on TEST:normal_dna successfully +[2020-11-26T14:28:12-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp3515c6nz/worker_log.txt +[2020-11-26T14:28:12-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-018i7hj0 +[2020-11-26T14:28:12-0800] [MainThread] [I] [toil.leader] Issued job 'mark_duplicates' kind-mark_duplicates/instance-sy_g4ltz with job batch system ID: 65 and cores: 1, disk: 1.3 G, and memory: 12.0 G +[2020-11-26T14:28:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp8lujtak0/worker_log.txt +[2020-11-26T14:28:29-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:28:29: Ran add_read_groups on TEST:tumor_dna successfully +[2020-11-26T14:28:30-0800] [MainThread] [I] [toil.leader] Job ended: 'add_readgroups' kind-add_readgroups/instance-5h5oumic +[2020-11-26T14:28:30-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-16cmbaqb with job batch system ID: 66 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:28:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpoxxu3cx0/worker_log.txt +[2020-11-26T14:28:30-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-16cmbaqb +[2020-11-26T14:28:30-0800] [MainThread] [I] [toil.leader] Issued job 'mark_duplicates' kind-mark_duplicates/instance-21qv1epf with job batch system ID: 67 and cores: 1, disk: 1.5 G, and memory: 12.0 G +[2020-11-26T14:28:31-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp0hmhnhlz/worker_log.txt +[2020-11-26T14:30:30-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:30:30: Ran mark_duplicates on TEST:normal_dna successfully +[2020-11-26T14:30:30-0800] [MainThread] [I] [toil.leader] Job ended: 'mark_duplicates' kind-mark_duplicates/instance-sy_g4ltz +[2020-11-26T14:30:30-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-z4yj3q9w with job batch system ID: 68 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:30:31-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp2kifbe3c/worker_log.txt +[2020-11-26T14:30:31-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-z4yj3q9w +[2020-11-26T14:30:31-0800] [MainThread] [I] [toil.leader] Issued job 'index_bamfile' kind-index_bamfile/instance-fz2ff11q with job batch system ID: 69 and cores: 1, disk: 360.6 M, and memory: 12.0 G +[2020-11-26T14:30:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpz0fo7i6r/worker_log.txt +[2020-11-26T14:30:55-0800] [MainThread] [I] [toil.leader] Job ended: 'index_bamfile' kind-index_bamfile/instance-fz2ff11q +[2020-11-26T14:30:55-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-mj9pw7z1 with job batch system ID: 70 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:30:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:30:55: Exporting files/for-job/kind-mark_duplicates/instance-sy_g4ltz/file-923eo14r/normal_dna_mkdup.bam to output location +[2020-11-26T14:30:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:30:55: Exporting files/for-job/kind-index_bamfile/instance-fz2ff11q/file-yr3ow4v6/normal_dna_fix_pg_sorted.bam.bai to output location +[2020-11-26T14:30:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:30:55: Ran samtools-index on TEST:normal_dna successfully +[2020-11-26T14:30:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpvcc88wzv/worker_log.txt +[2020-11-26T14:30:56-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-mj9pw7z1 +[2020-11-26T14:30:56-0800] [MainThread] [I] [toil.leader] Issued job 'index_bamfile' kind-Job/instance-z4yj3q9w with job batch system ID: 71 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:30:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpfya0tf58/worker_log.txt +[2020-11-26T14:30:57-0800] [MainThread] [I] [toil.leader] Job ended: 'index_bamfile' kind-Job/instance-z4yj3q9w +[2020-11-26T14:30:57-0800] [MainThread] [I] [toil.leader] Issued job 'mark_duplicates' kind-Job/instance-018i7hj0 with job batch system ID: 72 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:30:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpuf2tl4ed/worker_log.txt +[2020-11-26T14:30:58-0800] [MainThread] [I] [toil.leader] Job ended: 'mark_duplicates' kind-Job/instance-018i7hj0 +[2020-11-26T14:30:58-0800] [MainThread] [I] [toil.leader] Issued job 'add_readgroups' kind-Job/instance-dxink9kr with job batch system ID: 73 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:30:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpyl8azgz4/worker_log.txt +[2020-11-26T14:30:59-0800] [MainThread] [I] [toil.leader] Job ended: 'add_readgroups' kind-Job/instance-dxink9kr +[2020-11-26T14:30:59-0800] [MainThread] [I] [toil.leader] Issued job 'fix_bam_header' kind-Job/instance-kgtbwyfd with job batch system ID: 74 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:30:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpoohrz98e/worker_log.txt +[2020-11-26T14:30:59-0800] [MainThread] [I] [toil.leader] Job ended: 'fix_bam_header' kind-Job/instance-kgtbwyfd +[2020-11-26T14:30:59-0800] [MainThread] [I] [toil.leader] Issued job 'bam_conversion' kind-Job/instance-yuscfm9u with job batch system ID: 75 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:31:00-0800] [MainThread] [I] [toil.leader] Job ended: 'mark_duplicates' kind-mark_duplicates/instance-21qv1epf +[2020-11-26T14:31:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-5p3g4k17 with job batch system ID: 76 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:31:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp0cys2s56/worker_log.txt +[2020-11-26T14:31:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:31:00: Ran mark_duplicates on TEST:tumor_dna successfully +[2020-11-26T14:31:00-0800] [MainThread] [I] [toil.leader] Job ended: 'bam_conversion' kind-Job/instance-yuscfm9u +[2020-11-26T14:31:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-2093kxca with job batch system ID: 77 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:31:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp8ht42f2o/worker_log.txt +[2020-11-26T14:31:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp0weeem6_/worker_log.txt +[2020-11-26T14:31:01-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-2093kxca +[2020-11-26T14:31:01-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-5p3g4k17 +[2020-11-26T14:31:01-0800] [MainThread] [I] [toil.leader] Issued job 'index_bamfile' kind-index_bamfile/instance-069qv896 with job batch system ID: 78 and cores: 1, disk: 412.9 M, and memory: 12.0 G +[2020-11-26T14:31:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp42vk_ydq/worker_log.txt +[2020-11-26T14:31:21-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:31:21: Ran phlat on TEST:normal_dna successfully +[2020-11-26T14:31:22-0800] [MainThread] [I] [toil.leader] Job ended: 'run_phlat' kind-run_phlat/instance-8v93d6hk +[2020-11-26T14:31:22-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-f9i75eyk with job batch system ID: 79 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:31:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp7q__v4qo/worker_log.txt +[2020-11-26T14:31:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp6civqf7m/worker_log.txt +[2020-11-26T14:31:22-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-f9i75eyk +[2020-11-26T14:31:22-0800] [MainThread] [I] [toil.leader] Issued job 'merge_phlat_calls' kind-merge_phlat_calls/instance-h83z47yn with job batch system ID: 80 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-26T14:31:23-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpfucgvw2p/worker_log.txt +[2020-11-26T14:31:23-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_phlat_calls' kind-merge_phlat_calls/instance-h83z47yn +[2020-11-26T14:31:24-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:31:24: Merging Phlat calls +[2020-11-26T14:31:24-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:31:24: Exporting files/for-job/kind-merge_phlat_calls/instance-h83z47yn/file-bf12qdku/mhci_alleles.list to output location +[2020-11-26T14:31:24-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:31:24: Exporting files/for-job/kind-merge_phlat_calls/instance-h83z47yn/file-4kzwkoqf/mhcii_alleles.list to output location +[2020-11-26T14:31:48-0800] [MainThread] [I] [toil.leader] Job ended: 'index_bamfile' kind-index_bamfile/instance-069qv896 +[2020-11-26T14:31:48-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-dmj2tegs with job batch system ID: 81 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:31:49-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:31:49: Exporting files/for-job/kind-mark_duplicates/instance-21qv1epf/file-qoicb03g/tumor_dna_mkdup.bam to output location +[2020-11-26T14:31:49-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:31:49: Exporting files/for-job/kind-index_bamfile/instance-069qv896/file-t5ysa9ej/tumor_dna_fix_pg_sorted.bam.bai to output location +[2020-11-26T14:31:49-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:31:49: Ran samtools-index on TEST:tumor_dna successfully +[2020-11-26T14:31:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpgh7xed29/worker_log.txt +[2020-11-26T14:31:49-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-dmj2tegs +[2020-11-26T14:31:49-0800] [MainThread] [I] [toil.leader] Issued job 'index_bamfile' kind-Job/instance-5p3g4k17 with job batch system ID: 82 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:31:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp64c2bdvf/worker_log.txt +[2020-11-26T14:31:50-0800] [MainThread] [I] [toil.leader] Job ended: 'index_bamfile' kind-Job/instance-5p3g4k17 +[2020-11-26T14:31:50-0800] [MainThread] [I] [toil.leader] Issued job 'mark_duplicates' kind-Job/instance-16cmbaqb with job batch system ID: 83 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:31:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpm1s2l2r6/worker_log.txt +[2020-11-26T14:31:51-0800] [MainThread] [I] [toil.leader] Job ended: 'mark_duplicates' kind-Job/instance-16cmbaqb +[2020-11-26T14:31:51-0800] [MainThread] [I] [toil.leader] Issued job 'add_readgroups' kind-Job/instance-t810tc85 with job batch system ID: 84 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:31:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp5gwuindg/worker_log.txt +[2020-11-26T14:31:52-0800] [MainThread] [I] [toil.leader] Job ended: 'add_readgroups' kind-Job/instance-t810tc85 +[2020-11-26T14:31:52-0800] [MainThread] [I] [toil.leader] Issued job 'fix_bam_header' kind-Job/instance-f4hwh0wg with job batch system ID: 85 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:31:52-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpz895po6s/worker_log.txt +[2020-11-26T14:31:52-0800] [MainThread] [I] [toil.leader] Job ended: 'fix_bam_header' kind-Job/instance-f4hwh0wg +[2020-11-26T14:31:52-0800] [MainThread] [I] [toil.leader] Issued job 'bam_conversion' kind-Job/instance-_3_wmlwl with job batch system ID: 86 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:31:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmplfsnpa3q/worker_log.txt +[2020-11-26T14:31:53-0800] [MainThread] [I] [toil.leader] Job ended: 'bam_conversion' kind-Job/instance-_3_wmlwl +[2020-11-26T14:31:53-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-qb5fy02h with job batch system ID: 87 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:31:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp13wa99cd/worker_log.txt +[2020-11-26T14:31:54-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-qb5fy02h +[2020-11-26T14:31:54-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-pexfcemb with job batch system ID: 88 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-26T14:31:54-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-1hcsd04a with job batch system ID: 89 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:31:54-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-zx8np8ze with job batch system ID: 90 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:31:54-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-qlrl0tcr with job batch system ID: 91 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:31:54-0800] [MainThread] [I] [toil.leader] Issued job 'run_indel_caller' kind-run_indel_caller/instance-w96x5afy with job batch system ID: 92 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-26T14:31:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp_d27ic25/worker_log.txt +[2020-11-26T14:31:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpo08z1xw_/worker_log.txt +[2020-11-26T14:31:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpt9ou5_gn/worker_log.txt +[2020-11-26T14:31:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmphsrzyelj/worker_log.txt +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpmzpugxpg/worker_log.txt +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Job ended: 'run_indel_caller' kind-run_indel_caller/instance-w96x5afy +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-zx8np8ze +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-cez1709k with job batch system ID: 93 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-vbeka1dz with job batch system ID: 94 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-qlrl0tcr +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_strelka_full' kind-run_strelka_full/instance-_6r67yr7 with job batch system ID: 95 and cores: 25, disk: 6.0 G, and memory: 6.0 G +[2020-11-26T14:31:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:31:55: INDELs are currently unsupported.... Skipping. +[2020-11-26T14:31:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:31:55: Ran INDEL on TEST successfully +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-pexfcemb +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-r7zfh_mi with job batch system ID: 96 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-6o1f2_bm with job batch system ID: 97 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-cea7k1l1 with job batch system ID: 98 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-xpymqto9 with job batch system ID: 99 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-a7dqkhgr with job batch system ID: 100 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-ya95zc8d with job batch system ID: 101 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-r5_1g_p1 with job batch system ID: 102 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-42j05bv2 with job batch system ID: 103 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-0go3azmx with job batch system ID: 104 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-igzj4hlq with job batch system ID: 105 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-rrqzz7iy with job batch system ID: 106 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-9uhvfwte with job batch system ID: 107 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-1lifd6w0 with job batch system ID: 108 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-88cvjwqg with job batch system ID: 109 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-z_yip7u5 with job batch system ID: 110 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-964s4dzy with job batch system ID: 111 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-zqzduxxb with job batch system ID: 112 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-922g8ch1 with job batch system ID: 113 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-7v8w38dg with job batch system ID: 114 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-lbfk67uk with job batch system ID: 115 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-7mwxogrn with job batch system ID: 116 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-mvuk5vsv with job batch system ID: 117 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-b4xv7mk2 with job batch system ID: 118 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-hhrg60f3 with job batch system ID: 119 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-idxh2vdj with job batch system ID: 120 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-1hcsd04a +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-fwuy395_ with job batch system ID: 121 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-3it4bha6 with job batch system ID: 122 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-v8c8lue8 with job batch system ID: 123 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-j08l0kw9 with job batch system ID: 124 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-3x8ybfw1 with job batch system ID: 125 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-pmph1fk4 with job batch system ID: 126 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-ng6aqth7 with job batch system ID: 127 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-lxgneqv4 with job batch system ID: 128 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-9egl4j3h with job batch system ID: 129 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-ilpnlrnv with job batch system ID: 130 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-fj2gq4jg with job batch system ID: 131 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-x1ttz7h4 with job batch system ID: 132 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-4jvd65yl with job batch system ID: 133 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-m9ielfqx with job batch system ID: 134 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-8tw0u5y3 with job batch system ID: 135 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-4vny_tq_ with job batch system ID: 136 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-634x5hd7 with job batch system ID: 137 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-owi8ez8_ with job batch system ID: 138 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-0641pam7 with job batch system ID: 139 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-ubxl0ikt with job batch system ID: 140 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-muknk2u5 with job batch system ID: 141 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-ane1bnfa with job batch system ID: 142 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-w10nget5 with job batch system ID: 143 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-ywu8pvrz with job batch system ID: 144 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:55-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-6beeo86z with job batch system ID: 145 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:31:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpty_r92m0/worker_log.txt +[2020-11-26T14:31:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpdko9p5n7/worker_log.txt +[2020-11-26T14:31:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpyb6r6t01/worker_log.txt +[2020-11-26T14:31:56-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-cez1709k +[2020-11-26T14:31:56-0800] [MainThread] [I] [toil.leader] Issued job 'run_somaticsniper_full' kind-run_somaticsniper_full/instance-dxbjtncv with job batch system ID: 146 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:56-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-vbeka1dz +[2020-11-26T14:31:56-0800] [MainThread] [I] [toil.leader] Issued job 'run_pileup' kind-run_pileup/instance-f4ni_emz with job batch system ID: 147 and cores: 1, disk: 4.8 G, and memory: 6.0 G +[2020-11-26T14:31:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpq2s383o6/worker_log.txt +[2020-11-26T14:31:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp8tjgkmkr/worker_log.txt +[2020-11-26T14:31:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpp4pi6d1m/worker_log.txt +[2020-11-26T14:31:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp96n7sg0y/worker_log.txt +[2020-11-26T14:31:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpdoxnc9cu/worker_log.txt +[2020-11-26T14:31:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpcuhe05td/worker_log.txt +[2020-11-26T14:31:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp5t3ret2g/worker_log.txt +[2020-11-26T14:31:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpkcic0lzq/worker_log.txt +[2020-11-26T14:31:57-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-6o1f2_bm +[2020-11-26T14:31:57-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-tylsalqp with job batch system ID: 148 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpqdktv8jl/worker_log.txt +[2020-11-26T14:31:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpqb1t8ph3/worker_log.txt +[2020-11-26T14:31:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp40j1c50s/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpwj97kvj9/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp1dvewh1b/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-xpymqto9 +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-j4dbuxu_ with job batch system ID: 149 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpyrgese6n/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpvvhozup8/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp58_d_n5a/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp1otptl_f/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpccvlc_os/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-ya95zc8d +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9gv8fvfc with job batch system ID: 150 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpm1jetr2u/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmph8vqo1l7/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp9pmny8xl/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpzikv80cp/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-r7zfh_mi +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ejsu5bhp with job batch system ID: 151 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpp_xa6wxb/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-cea7k1l1 +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-88dbdpi7 with job batch system ID: 152 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpn0vnfkek/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp6x4gonmk/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-42j05bv2 +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-80izrmlo with job batch system ID: 153 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-r5_1g_p1 +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-68lg3d35 with job batch system ID: 154 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-a7dqkhgr +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-8fvafg6t with job batch system ID: 155 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpeknhuzj2/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-igzj4hlq +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ccch6489 with job batch system ID: 156 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-rrqzz7iy +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-sr1sdn33 with job batch system ID: 157 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp1hg47450/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp9ay52vgr/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-88cvjwqg +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp2c1cecl1/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-_24anocq with job batch system ID: 158 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp5ygeiqr2/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpsnvwjlo7/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp3xz5myil/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpr0i345xg/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpzp0ketfk/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpxvwwvk3q/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpgpoa3nm3/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmph2wuh_42/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-0go3azmx +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-8zydao1p with job batch system ID: 159 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpf9i3g8wi/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpad_7zlyv/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpeqyrid_y/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpn_6wyt_c/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-9uhvfwte +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-wuk0pxrz with job batch system ID: 160 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmplbvkcx9r/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpjl49j58c/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpxmnybqc2/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp210k8o4d/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpgxsgrcps/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpy6sowf1e/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpxv82g1lx/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpuzuz3j6k/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp73aoqg3f/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpm9xu26_v/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpm3shkkjb/worker_log.txt +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-1lifd6w0 +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fm3nb956 with job batch system ID: 161 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-z_yip7u5 +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-w6_nim9d with job batch system ID: 162 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-964s4dzy +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ybsrbqhv with job batch system ID: 163 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-zqzduxxb +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-34i0dvlp with job batch system ID: 164 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-7v8w38dg +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-bjpeedgo with job batch system ID: 165 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-922g8ch1 +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jaumkjks with job batch system ID: 166 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-7mwxogrn +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mdc_bcxs with job batch system ID: 167 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-mvuk5vsv +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-kri0peu0 with job batch system ID: 168 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-lbfk67uk +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-sz8e594a with job batch system ID: 169 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-hhrg60f3 +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-rev31ibh with job batch system ID: 170 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-idxh2vdj +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-wspvzxc0 with job batch system ID: 171 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-v8c8lue8 +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-d3w1qts3 with job batch system ID: 172 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-b4xv7mk2 +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gve8gkcu with job batch system ID: 173 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-fwuy395_ +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-fqpikpzh with job batch system ID: 174 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-lxgneqv4 +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-byzo9kk1 with job batch system ID: 175 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-muknk2u5 +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-1u_z0m6q with job batch system ID: 176 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-3x8ybfw1 +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-4fxrx92n with job batch system ID: 177 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-j08l0kw9 +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-zaeuogy5 with job batch system ID: 178 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-pmph1fk4 +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-99xsqvp7 with job batch system ID: 179 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-fj2gq4jg +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-v78605d0 with job batch system ID: 180 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-9egl4j3h +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-8_0cmk11 with job batch system ID: 181 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-3it4bha6 +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-cu5_rm6e with job batch system ID: 182 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-ng6aqth7 +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-xbx3mpev with job batch system ID: 183 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-x1ttz7h4 +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-9efej_90 with job batch system ID: 184 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-m9ielfqx +[2020-11-26T14:31:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-adb96d1d with job batch system ID: 185 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-ilpnlrnv +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-icema6p3 with job batch system ID: 186 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-4jvd65yl +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-826tlp82 with job batch system ID: 187 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp4vh4m9wf/worker_log.txt +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-8tw0u5y3 +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-c7810uxm with job batch system ID: 188 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-4vny_tq_ +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-3h8rmwuf with job batch system ID: 189 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-634x5hd7 +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-7d6qx_or with job batch system ID: 190 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-owi8ez8_ +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-5k6f7w6i with job batch system ID: 191 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-0641pam7 +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-zoygzglq with job batch system ID: 192 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-ubxl0ikt +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-dkbm3iy3 with job batch system ID: 193 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-ane1bnfa +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-hdui31xe with job batch system ID: 194 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-w10nget5 +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-cy72oyxc with job batch system ID: 195 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-ywu8pvrz +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-q7ze545r with job batch system ID: 196 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-6beeo86z +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-q529jly0 with job batch system ID: 197 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpu63u4b5n/worker_log.txt +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpssxc4z6u/worker_log.txt +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp6znh9c5d/worker_log.txt +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp38eo5tc8/worker_log.txt +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpn5di0y_q/worker_log.txt +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp26716zfm/worker_log.txt +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp6y_rne88/worker_log.txt +[2020-11-26T14:31:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp4jnqtfgc/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpwzfh4giw/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpmezyurco/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpa481vsxt/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpmieflr0j/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpjkwpo3wq/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpsm_ik4lk/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpwv14r6ml/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpodchwdk8/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp4hy6pfxy/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpxpf0_lkt/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpozkf4rfk/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmptbq103hu/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpw33neg2b/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp_sm9kphn/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpxj4ytj7j/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpyzblezwz/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp328wskp7/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp2vxcawl_/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmppl4t5pi9/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpx0h0e9cg/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp8o728ykx/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp_ehq343_/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpwh_obrv1/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp75zju3cd/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpueyeze76/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmps89v9vga/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmphtrmmcu8/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmplt_t56ti/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpoaalo2f4/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpw9s6uz_7/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpla0vuu77/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpfchukia8/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmplkdtvg8u/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpk2gu99ix/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpqjxwrdxh/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp17h7ph18/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpu5_n81at/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp5c9gin1j/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmproon44bh/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpf08wuabo/worker_log.txt +[2020-11-26T14:32:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp6r0ssk88/worker_log.txt +[2020-11-26T14:54:06-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:06: Ran MuSE on TEST:chr22 successfully +[2020-11-26T14:54:06-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:06: Ran MuSE on TEST:chrM successfully +[2020-11-26T14:54:06-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-q529jly0 +[2020-11-26T14:54:06-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-tf0p7utp with job batch system ID: 198 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:07-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-hdui31xe +[2020-11-26T14:54:07-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-rrdp8cl7 with job batch system ID: 199 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpsply_luy/worker_log.txt +[2020-11-26T14:54:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp_ylvs3zh/worker_log.txt +[2020-11-26T14:54:08-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-tf0p7utp +[2020-11-26T14:54:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-j9ii0mwr with job batch system ID: 200 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:08-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-rrdp8cl7 +[2020-11-26T14:54:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-vvi6pgcn with job batch system ID: 201 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpw5ca71t_/worker_log.txt +[2020-11-26T14:54:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpihz918je/worker_log.txt +[2020-11-26T14:54:27-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:27: Ran MuSE on TEST:chr12 successfully +[2020-11-26T14:54:28-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-9efej_90 +[2020-11-26T14:54:28-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-qx7svqtx with job batch system ID: 202 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:29-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpkwn0wzmp/worker_log.txt +[2020-11-26T14:54:29-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-qx7svqtx +[2020-11-26T14:54:29-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-9wd_y_pv with job batch system ID: 203 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpiviwhx6m/worker_log.txt +[2020-11-26T14:54:30-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:30: Ran MuSE on TEST:chr15 successfully +[2020-11-26T14:54:31-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-c7810uxm +[2020-11-26T14:54:31-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-i4_2msct with job batch system ID: 204 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:31: Ran MuSE on TEST:chr21 successfully +[2020-11-26T14:54:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:31: Ran MuSE on TEST:chrY successfully +[2020-11-26T14:54:32-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-1u_z0m6q +[2020-11-26T14:54:32-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-z2t6r5gk with job batch system ID: 205 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmptdkuaqob/worker_log.txt +[2020-11-26T14:54:32-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-q7ze545r +[2020-11-26T14:54:32-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-t5nqzund with job batch system ID: 206 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:32-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:32: Ran MuSE on TEST:chr18 successfully +[2020-11-26T14:54:32-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:32: Ran MuSE on TEST:chr6 successfully +[2020-11-26T14:54:32-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-i4_2msct +[2020-11-26T14:54:32-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-n1u6uby_ with job batch system ID: 207 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp03i1ipwx/worker_log.txt +[2020-11-26T14:54:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:33: Ran MuSE on TEST:chr14 successfully +[2020-11-26T14:54:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpeid2gl6e/worker_log.txt +[2020-11-26T14:54:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpcld3undv/worker_log.txt +[2020-11-26T14:54:33-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-z2t6r5gk +[2020-11-26T14:54:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-hnhl441_ with job batch system ID: 208 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:33-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-5k6f7w6i +[2020-11-26T14:54:33-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-q8o77o71 with job batch system ID: 209 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:33-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-t5nqzund +[2020-11-26T14:54:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-oc2t0kj_ with job batch system ID: 210 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:33: Ran MuSE on TEST:chr16 successfully +[2020-11-26T14:54:33-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-99xsqvp7 +[2020-11-26T14:54:33-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-agheoc17 with job batch system ID: 211 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:33-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-adb96d1d +[2020-11-26T14:54:33-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-nbubmf79 with job batch system ID: 212 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:34-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:34: Ran MuSE on TEST:chr8 successfully +[2020-11-26T14:54:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpualhgpsz/worker_log.txt +[2020-11-26T14:54:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpruso65gz/worker_log.txt +[2020-11-26T14:54:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmphgm457x8/worker_log.txt +[2020-11-26T14:54:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmposb5y6sq/worker_log.txt +[2020-11-26T14:54:34-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-q8o77o71 +[2020-11-26T14:54:34-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-vs45a200 with job batch system ID: 213 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmppe3ksi8k/worker_log.txt +[2020-11-26T14:54:34-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:34: Ran MuSE on TEST:chr2 successfully +[2020-11-26T14:54:34-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-3h8rmwuf +[2020-11-26T14:54:34-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-e7tctepi with job batch system ID: 214 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:34-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-agheoc17 +[2020-11-26T14:54:34-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-q52acj5t with job batch system ID: 215 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:34-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-byzo9kk1 +[2020-11-26T14:54:34-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-uf493142 with job batch system ID: 216 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:35-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-nbubmf79 +[2020-11-26T14:54:35-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-z2blc_z0 with job batch system ID: 217 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:35: Ran MuSE on TEST:chr1 successfully +[2020-11-26T14:54:35-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-cu5_rm6e +[2020-11-26T14:54:35-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-54b9nv8g with job batch system ID: 218 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpy7esdeju/worker_log.txt +[2020-11-26T14:54:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpojocphy6/worker_log.txt +[2020-11-26T14:54:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:35: Ran MuSE on TEST:chr11 successfully +[2020-11-26T14:54:35-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-e7tctepi +[2020-11-26T14:54:35-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-xrae656a with job batch system ID: 219 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp113zjt1z/worker_log.txt +[2020-11-26T14:54:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpeg0yyc5s/worker_log.txt +[2020-11-26T14:54:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpqvbeip2u/worker_log.txt +[2020-11-26T14:54:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpxsk9fv7s/worker_log.txt +[2020-11-26T14:54:36-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:36: Ran MuSE on TEST:chr19 successfully +[2020-11-26T14:54:36-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-fqpikpzh +[2020-11-26T14:54:36-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-l1v_5aqb with job batch system ID: 220 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:36-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-uf493142 +[2020-11-26T14:54:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-qwau497k with job batch system ID: 221 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:36-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-v78605d0 +[2020-11-26T14:54:36-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-0bz9s3zz with job batch system ID: 222 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:36-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-54b9nv8g +[2020-11-26T14:54:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-mp5gbc4z with job batch system ID: 223 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpdswscqeh/worker_log.txt +[2020-11-26T14:54:36-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:36: Ran MuSE on TEST:chrX successfully +[2020-11-26T14:54:36-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-zoygzglq +[2020-11-26T14:54:36-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-k3zvl93d with job batch system ID: 224 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpa6kw7ztq/worker_log.txt +[2020-11-26T14:54:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp1uzsjq2j/worker_log.txt +[2020-11-26T14:54:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpqh8eh3t9/worker_log.txt +[2020-11-26T14:54:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpmx8x7_x4/worker_log.txt +[2020-11-26T14:54:37-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-cy72oyxc +[2020-11-26T14:54:37-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-mn9os4nn with job batch system ID: 225 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:37: Ran MuSE on TEST:chr3 successfully +[2020-11-26T14:54:37-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-l1v_5aqb +[2020-11-26T14:54:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-pqk0yuup with job batch system ID: 226 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:37-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-0bz9s3zz +[2020-11-26T14:54:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-rsi_i_is with job batch system ID: 227 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpwa0qeeyv/worker_log.txt +[2020-11-26T14:54:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:37: Ran MuSE on TEST:chr4 successfully +[2020-11-26T14:54:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:37: Ran MuSE on TEST:chr9 successfully +[2020-11-26T14:54:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpoblzp0fg/worker_log.txt +[2020-11-26T14:54:38-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-d3w1qts3 +[2020-11-26T14:54:38-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-2uoht33e with job batch system ID: 228 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:38-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-k3zvl93d +[2020-11-26T14:54:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-7z04dui9 with job batch system ID: 229 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:38-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-8_0cmk11 +[2020-11-26T14:54:38-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-5l362nqc with job batch system ID: 230 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:38: Ran MuSE on TEST:chr7 successfully +[2020-11-26T14:54:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp8xginyii/worker_log.txt +[2020-11-26T14:54:38-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-mn9os4nn +[2020-11-26T14:54:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-ltc10ler with job batch system ID: 231 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmphteqqtzw/worker_log.txt +[2020-11-26T14:54:38-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-zaeuogy5 +[2020-11-26T14:54:38-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-umrtd9l5 with job batch system ID: 232 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:38: Ran MuSE on TEST:chr5 successfully +[2020-11-26T14:54:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp6v63znp6/worker_log.txt +[2020-11-26T14:54:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpyj4zroiq/worker_log.txt +[2020-11-26T14:54:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpfkpwqrf8/worker_log.txt +[2020-11-26T14:54:39-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-xbx3mpev +[2020-11-26T14:54:39-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ki2faqc3 with job batch system ID: 233 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:39: Ran MuSE on TEST:chr13 successfully +[2020-11-26T14:54:39-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-2uoht33e +[2020-11-26T14:54:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-qrufkbap with job batch system ID: 234 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpo01ihlon/worker_log.txt +[2020-11-26T14:54:39-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-5l362nqc +[2020-11-26T14:54:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-hz6rgj9y with job batch system ID: 235 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmptklucu1d/worker_log.txt +[2020-11-26T14:54:39-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-4fxrx92n +[2020-11-26T14:54:39-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-9ve_r5dx with job batch system ID: 236 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:39: Ran MuSE on TEST:chr17 successfully +[2020-11-26T14:54:39-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-umrtd9l5 +[2020-11-26T14:54:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-1g2mf0g0 with job batch system ID: 237 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp7c567z_s/worker_log.txt +[2020-11-26T14:54:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp1iew7_1h/worker_log.txt +[2020-11-26T14:54:40-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-826tlp82 +[2020-11-26T14:54:40-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-_et1cn27 with job batch system ID: 238 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp36lywwgp/worker_log.txt +[2020-11-26T14:54:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpeye2h2it/worker_log.txt +[2020-11-26T14:54:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:40: Ran MuSE on TEST:chr20 successfully +[2020-11-26T14:54:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:40: Ran MuSE on TEST:chr10 successfully +[2020-11-26T14:54:40-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ki2faqc3 +[2020-11-26T14:54:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-t8y6umyh with job batch system ID: 239 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:40-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-9ve_r5dx +[2020-11-26T14:54:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-vc4g0bse with job batch system ID: 240 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:40-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-7d6qx_or +[2020-11-26T14:54:40-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-99ahll_a with job batch system ID: 241 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp8ix1c84e/worker_log.txt +[2020-11-26T14:54:40-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-icema6p3 +[2020-11-26T14:54:40-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-woladh6f with job batch system ID: 242 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpma2zgtgk/worker_log.txt +[2020-11-26T14:54:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-dkbm3iy3 +[2020-11-26T14:54:41-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-yoi9d_qi with job batch system ID: 243 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpe2_nmw3n/worker_log.txt +[2020-11-26T14:54:41-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-_et1cn27 +[2020-11-26T14:54:41-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-96gq6bvg with job batch system ID: 244 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpee_vusji/worker_log.txt +[2020-11-26T14:54:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp2bvozh3v/worker_log.txt +[2020-11-26T14:54:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpk1gsmobl/worker_log.txt +[2020-11-26T14:54:42-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-woladh6f +[2020-11-26T14:54:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-xd04sblt with job batch system ID: 245 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:42-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-99ahll_a +[2020-11-26T14:54:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-94251a3_ with job batch system ID: 246 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpc14sv6fn/worker_log.txt +[2020-11-26T14:54:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpr7jedazd/worker_log.txt +[2020-11-26T14:54:42-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-yoi9d_qi +[2020-11-26T14:54:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-fd905ar_ with job batch system ID: 247 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-26T14:54:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpic77logw/worker_log.txt +[2020-11-26T14:54:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpkvbp5c3c/worker_log.txt +[2020-11-26T14:54:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:42: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-vvi6pgcn/file-fmalg9k5/chr22.vcf to output location +[2020-11-26T14:54:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:42: Ran MuSE sump on TEST:chr22 successfully +[2020-11-26T14:54:42-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-vvi6pgcn +[2020-11-26T14:54:42-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-8s7iniik with job batch system ID: 248 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:43-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-n1u6uby_ +[2020-11-26T14:54:43-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ihcak3fc with job batch system ID: 249 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:43-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-vs45a200 +[2020-11-26T14:54:43-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-zl_flmtd with job batch system ID: 250 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:43-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-oc2t0kj_ +[2020-11-26T14:54:43-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ufemdu9s with job batch system ID: 251 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:43-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-hnhl441_ +[2020-11-26T14:54:43-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-z2w3fxxe with job batch system ID: 252 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpzzrkeeal/worker_log.txt +[2020-11-26T14:54:43-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-9wd_y_pv +[2020-11-26T14:54:43-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-hox7uzu5 with job batch system ID: 253 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:43-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-j9ii0mwr +[2020-11-26T14:54:43-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-r8w7rgom with job batch system ID: 254 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:43: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-vs45a200/file-hna5ht4p/chr18.vcf to output location +[2020-11-26T14:54:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:43: Ran MuSE sump on TEST:chr18 successfully +[2020-11-26T14:54:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:43: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-n1u6uby_/file-uxgqdteg/chr15.vcf to output location +[2020-11-26T14:54:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:43: Ran MuSE sump on TEST:chr15 successfully +[2020-11-26T14:54:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:43: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-oc2t0kj_/file-_73kgehx/chrY.vcf to output location +[2020-11-26T14:54:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:43: Ran MuSE sump on TEST:chrY successfully +[2020-11-26T14:54:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:43: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-hnhl441_/file-86h3agkl/chr21.vcf to output location +[2020-11-26T14:54:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:43: Ran MuSE sump on TEST:chr21 successfully +[2020-11-26T14:54:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:43: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-9wd_y_pv/file-jl3ne1ck/chr12.vcf to output location +[2020-11-26T14:54:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:43: Ran MuSE sump on TEST:chr12 successfully +[2020-11-26T14:54:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:43: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-j9ii0mwr/file-wzfjxigg/chrM.vcf to output location +[2020-11-26T14:54:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:43: Ran MuSE sump on TEST:chrM successfully +[2020-11-26T14:54:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpv38sy466/worker_log.txt +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpfin3ldz_/worker_log.txt +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-8s7iniik +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-rrdp8cl7 with job batch system ID: 255 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpkp0mgtyv/worker_log.txt +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpehj8c0sv/worker_log.txt +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmps71g5p2c/worker_log.txt +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp9zv5_z7h/worker_log.txt +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-z2w3fxxe +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-z2t6r5gk with job batch system ID: 256 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpw_7i6vkz/worker_log.txt +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ufemdu9s +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-t5nqzund with job batch system ID: 257 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ihcak3fc +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-i4_2msct with job batch system ID: 258 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-hox7uzu5 +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-qx7svqtx with job batch system ID: 259 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-zl_flmtd +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-q8o77o71 with job batch system ID: 260 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-xrae656a +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-_jfntpiz with job batch system ID: 261 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-r8w7rgom +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-tf0p7utp with job batch system ID: 262 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:44-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:44: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-xrae656a/file-x1fis71p/chr16.vcf to output location +[2020-11-26T14:54:44-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:44: Ran MuSE sump on TEST:chr16 successfully +[2020-11-26T14:54:44-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:44: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-q52acj5t/file-r8ud8ds5/chr6.vcf to output location +[2020-11-26T14:54:44-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:44: Ran MuSE sump on TEST:chr6 successfully +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-q52acj5t +[2020-11-26T14:54:44-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-vavxapnv with job batch system ID: 263 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpkv31f4el/worker_log.txt +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-qwau497k +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-aawoty0m with job batch system ID: 264 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-rrdp8cl7 +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-ane1bnfa with job batch system ID: 265 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp4axf60hr/worker_log.txt +[2020-11-26T14:54:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:45: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-z2blc_z0/file-6owqxr_p/chr14.vcf to output location +[2020-11-26T14:54:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:45: Ran MuSE sump on TEST:chr14 successfully +[2020-11-26T14:54:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:45: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-qwau497k/file-tvki8paj/chr8.vcf to output location +[2020-11-26T14:54:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:45: Ran MuSE sump on TEST:chr8 successfully +[2020-11-26T14:54:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:45: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-pqk0yuup/file-9p1d_ki0/chr1.vcf to output location +[2020-11-26T14:54:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:45: Ran MuSE sump on TEST:chr1 successfully +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-z2t6r5gk +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-muknk2u5 with job batch system ID: 266 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-pqk0yuup +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-o6kta9bp with job batch system ID: 267 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpah4gd9vv/worker_log.txt +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-z2blc_z0 +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-50tipadb with job batch system ID: 268 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpn4bk7duo/worker_log.txt +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp2_dq0ndr/worker_log.txt +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-i4_2msct +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp6eypqdt3/worker_log.txt +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-8tw0u5y3 with job batch system ID: 269 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-t5nqzund +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-ywu8pvrz with job batch system ID: 270 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-mp5gbc4z +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-rgftzcv3 with job batch system ID: 271 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpwesn_wpz/worker_log.txt +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-qx7svqtx +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-x1ttz7h4 with job batch system ID: 272 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-q8o77o71 +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-owi8ez8_ with job batch system ID: 273 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpc4gp8_qy/worker_log.txt +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-_jfntpiz +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-e7tctepi with job batch system ID: 274 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-tf0p7utp +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-6beeo86z with job batch system ID: 275 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:45: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-mp5gbc4z/file-ssv5myrm/chr2.vcf to output location +[2020-11-26T14:54:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:45: Ran MuSE sump on TEST:chr2 successfully +[2020-11-26T14:54:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpgxmk9_rf/worker_log.txt +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-vavxapnv +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-agheoc17 with job batch system ID: 276 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpdp03e3y0/worker_log.txt +[2020-11-26T14:54:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:46: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-rsi_i_is/file-k4k320fe/chr11.vcf to output location +[2020-11-26T14:54:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:46: Ran MuSE sump on TEST:chr11 successfully +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-rsi_i_is +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-8mwkntm3 with job batch system ID: 277 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-aawoty0m +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-uf493142 with job batch system ID: 278 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpmsftfrmb/worker_log.txt +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpgugad3v1/worker_log.txt +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpvnvf3pir/worker_log.txt +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpkujgs3qx/worker_log.txt +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-qrufkbap +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-o0b1_nig with job batch system ID: 279 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-7z04dui9 +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ttlbb_kc with job batch system ID: 280 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-hz6rgj9y +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-cp0799sa with job batch system ID: 281 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-ltc10ler +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-gwwq8rsq with job batch system ID: 282 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-ane1bnfa +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-muknk2u5 +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-o6kta9bp +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-l1v_5aqb with job batch system ID: 283 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp8t7pz77k/worker_log.txt +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-50tipadb +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-nbubmf79 with job batch system ID: 284 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpbhrsc618/worker_log.txt +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpvk1pfnyj/worker_log.txt +[2020-11-26T14:54:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-8tw0u5y3 +[2020-11-26T14:54:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:46: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-qrufkbap/file-iy6ah1h4/chr3.vcf to output location +[2020-11-26T14:54:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:46: Ran MuSE sump on TEST:chr3 successfully +[2020-11-26T14:54:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:46: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-ltc10ler/file-5pbtiq5r/chrX.vcf to output location +[2020-11-26T14:54:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:46: Ran MuSE sump on TEST:chrX successfully +[2020-11-26T14:54:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:46: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-7z04dui9/file-bdfiw0vq/chr19.vcf to output location +[2020-11-26T14:54:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:46: Ran MuSE sump on TEST:chr19 successfully +[2020-11-26T14:54:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:46: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-hz6rgj9y/file-p9ysnkuj/chr9.vcf to output location +[2020-11-26T14:54:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:46: Ran MuSE sump on TEST:chr9 successfully +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpdhiuxazl/worker_log.txt +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpa2vbt8ge/worker_log.txt +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-rgftzcv3 +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-54b9nv8g with job batch system ID: 285 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpsbtcfhy1/worker_log.txt +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-ywu8pvrz +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-owi8ez8_ +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-x1ttz7h4 +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpqurkmxhl/worker_log.txt +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-e7tctepi +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-4vny_tq_ with job batch system ID: 286 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-6beeo86z +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp9nj_9uk4/worker_log.txt +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-agheoc17 +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-pmph1fk4 with job batch system ID: 287 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpb06mko4x/worker_log.txt +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpnmf58kpc/worker_log.txt +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpmta5y0v0/worker_log.txt +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpzncg_zuy/worker_log.txt +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp_iq_8a7l/worker_log.txt +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpu2pnuwhc/worker_log.txt +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-uf493142 +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-lxgneqv4 with job batch system ID: 288 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-l1v_5aqb +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-fwuy395_ with job batch system ID: 289 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmptud95hoy/worker_log.txt +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-8mwkntm3 +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-0bz9s3zz with job batch system ID: 290 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-cp0799sa +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-5l362nqc with job batch system ID: 291 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ttlbb_kc +[2020-11-26T14:54:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-k3zvl93d with job batch system ID: 292 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpnco991bd/worker_log.txt +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpkzuvnr9i/worker_log.txt +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-o0b1_nig +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-2uoht33e with job batch system ID: 293 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-nbubmf79 +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-m9ielfqx with job batch system ID: 294 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-54b9nv8g +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-3it4bha6 with job batch system ID: 295 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-gwwq8rsq +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-mn9os4nn with job batch system ID: 296 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpxvarktd1/worker_log.txt +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-4vny_tq_ +[2020-11-26T14:54:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:48: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-96gq6bvg/file-m2b04j4x/chr13.vcf to output location +[2020-11-26T14:54:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:48: Ran MuSE sump on TEST:chr13 successfully +[2020-11-26T14:54:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:48: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-vc4g0bse/file-1cpvfyuj/chr5.vcf to output location +[2020-11-26T14:54:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:48: Ran MuSE sump on TEST:chr5 successfully +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-96gq6bvg +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-028f9r4b with job batch system ID: 297 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpo76ugo9l/worker_log.txt +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-1g2mf0g0 +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-bd2wnh_l with job batch system ID: 298 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-vc4g0bse +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-uff3vefa with job batch system ID: 299 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-pmph1fk4 +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-xd04sblt +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-89nqw057 with job batch system ID: 300 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-t8y6umyh +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-_t8f5gbd with job batch system ID: 301 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp0oq60ql8/worker_log.txt +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpveiocpmv/worker_log.txt +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpmc0q13l0/worker_log.txt +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpvlno6l1e/worker_log.txt +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-fwuy395_ +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-lxgneqv4 +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpw_ysuvu3/worker_log.txt +[2020-11-26T14:54:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:48: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-xd04sblt/file-1cz2pl81/chr10.vcf to output location +[2020-11-26T14:54:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:48: Ran MuSE sump on TEST:chr10 successfully +[2020-11-26T14:54:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:48: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-1g2mf0g0/file-33lf6w2q/chr4.vcf to output location +[2020-11-26T14:54:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:48: Ran MuSE sump on TEST:chr4 successfully +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-5l362nqc +[2020-11-26T14:54:48-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-9egl4j3h with job batch system ID: 302 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:49-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:48: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-t8y6umyh/file-9za85rq8/chr7.vcf to output location +[2020-11-26T14:54:49-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:48: Ran MuSE sump on TEST:chr7 successfully +[2020-11-26T14:54:49-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:48: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-94251a3_/file-3az5vdw9/chr17.vcf to output location +[2020-11-26T14:54:49-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:49: Ran MuSE sump on TEST:chr17 successfully +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-94251a3_ +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-jbtxui_o with job batch system ID: 303 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpzj2rrbx1/worker_log.txt +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-k3zvl93d +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-0641pam7 with job batch system ID: 304 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-0bz9s3zz +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-fj2gq4jg with job batch system ID: 305 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpr3iw0q7v/worker_log.txt +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp6d0neidw/worker_log.txt +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-fd905ar_ +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ikmf3q8r with job batch system ID: 306 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-2uoht33e +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-v8c8lue8 with job batch system ID: 307 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-m9ielfqx +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-3it4bha6 +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp8kmgmqxg/worker_log.txt +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-mn9os4nn +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-w10nget5 with job batch system ID: 308 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:49-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:49: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-fd905ar_/file-qsyqvb4s/chr20.vcf to output location +[2020-11-26T14:54:49-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:54:49: Ran MuSE sump on TEST:chr20 successfully +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp3w9j6weq/worker_log.txt +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpvp1h5y5z/worker_log.txt +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpfxvzeqmp/worker_log.txt +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-028f9r4b +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-_et1cn27 with job batch system ID: 309 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-uff3vefa +[2020-11-26T14:54:49-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-9ve_r5dx with job batch system ID: 310 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-bd2wnh_l +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-umrtd9l5 with job batch system ID: 311 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpfsmg3ht5/worker_log.txt +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp0pwxljte/worker_log.txt +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-89nqw057 +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-woladh6f with job batch system ID: 312 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpi6911cug/worker_log.txt +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp77b0jq9y/worker_log.txt +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-fj2gq4jg +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-_t8f5gbd +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-ki2faqc3 with job batch system ID: 313 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpmtrno3si/worker_log.txt +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-jbtxui_o +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-99ahll_a with job batch system ID: 314 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-9egl4j3h +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp7xa6yg7a/worker_log.txt +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpiq3n0iv3/worker_log.txt +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpq675o9n3/worker_log.txt +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-v8c8lue8 +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-0641pam7 +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ikmf3q8r +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-yoi9d_qi with job batch system ID: 315 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp7sxr_0g9/worker_log.txt +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpde1jm7tx/worker_log.txt +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp_foqahnd/worker_log.txt +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-w10nget5 +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-umrtd9l5 +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-j08l0kw9 with job batch system ID: 316 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-_et1cn27 +[2020-11-26T14:54:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-4jvd65yl with job batch system ID: 317 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp5ucd1ucr/worker_log.txt +[2020-11-26T14:54:51-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-9ve_r5dx +[2020-11-26T14:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-3x8ybfw1 with job batch system ID: 318 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpsyxktb1c/worker_log.txt +[2020-11-26T14:54:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp_0avku2z/worker_log.txt +[2020-11-26T14:54:51-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-woladh6f +[2020-11-26T14:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-ilpnlrnv with job batch system ID: 319 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:51-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-ki2faqc3 +[2020-11-26T14:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-ng6aqth7 with job batch system ID: 320 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpj76jgmpd/worker_log.txt +[2020-11-26T14:54:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpjr_9h8p7/worker_log.txt +[2020-11-26T14:54:51-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-yoi9d_qi +[2020-11-26T14:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-ubxl0ikt with job batch system ID: 321 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:51-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-99ahll_a +[2020-11-26T14:54:51-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-634x5hd7 with job batch system ID: 322 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:54:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpf6vowhrh/worker_log.txt +[2020-11-26T14:54:51-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-4jvd65yl +[2020-11-26T14:54:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmphwlj8buy/worker_log.txt +[2020-11-26T14:54:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpy7fqfeqj/worker_log.txt +[2020-11-26T14:54:52-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-3x8ybfw1 +[2020-11-26T14:54:52-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpnef5kzq5/worker_log.txt +[2020-11-26T14:54:52-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmphsdwk_wq/worker_log.txt +[2020-11-26T14:54:52-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-ilpnlrnv +[2020-11-26T14:54:52-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-j08l0kw9 +[2020-11-26T14:54:52-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-ng6aqth7 +[2020-11-26T14:54:52-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp8u1os1no/worker_log.txt +[2020-11-26T14:54:52-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-ubxl0ikt +[2020-11-26T14:54:52-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpyscits9y/worker_log.txt +[2020-11-26T14:54:52-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-634x5hd7 +[2020-11-26T14:54:52-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-7n6lth9u with job batch system ID: 323 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:54:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpayupyxvq/worker_log.txt +[2020-11-26T14:54:53-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-7n6lth9u +[2020-11-26T14:55:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:55:00: Ran samtools pileup on TEST successfully +[2020-11-26T14:55:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_pileup' kind-run_pileup/instance-f4ni_emz +[2020-11-26T14:55:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-98y719jf with job batch system ID: 324 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:55:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp80dcg2nd/worker_log.txt +[2020-11-26T14:55:01-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-98y719jf +[2020-11-26T14:56:11-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:11: Ran SomaticSniper on TEST successfully +[2020-11-26T14:56:11-0800] [MainThread] [I] [toil.leader] Job ended: 'run_somaticsniper_full' kind-run_somaticsniper_full/instance-dxbjtncv +[2020-11-26T14:56:11-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-1se_o02s with job batch system ID: 325 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:56:12-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpkwf9eu6h/worker_log.txt +[2020-11-26T14:56:12-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-1se_o02s +[2020-11-26T14:56:12-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-7a1dtxge with job batch system ID: 326 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:56:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp2l33i603/worker_log.txt +[2020-11-26T14:56:13-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-7a1dtxge +[2020-11-26T14:56:13-0800] [MainThread] [I] [toil.leader] Issued job 'filter_somaticsniper' kind-filter_somaticsniper/instance-obid9nr_ with job batch system ID: 327 and cores: 1, disk: 4.8 G, and memory: 6.0 G +[2020-11-26T14:56:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpsfubxj5u/worker_log.txt +[2020-11-26T14:56:49-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:49: Ran strelka on TEST successfully +[2020-11-26T14:56:49-0800] [MainThread] [I] [toil.leader] Job ended: 'run_strelka_full' kind-run_strelka_full/instance-_6r67yr7 +[2020-11-26T14:56:49-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-v67dee47 with job batch system ID: 328 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:56:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpc1skmeco/worker_log.txt +[2020-11-26T14:56:50-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-v67dee47 +[2020-11-26T14:56:50-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-7e2mu_4f with job batch system ID: 329 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:56:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpv21l3075/worker_log.txt +[2020-11-26T14:56:51-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-7e2mu_4f +[2020-11-26T14:56:51-0800] [MainThread] [I] [toil.leader] Issued job 'unmerge' kind-unmerge/instance-ritimmsk with job batch system ID: 330 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:56:51-0800] [MainThread] [I] [toil.leader] Issued job 'unmerge' kind-unmerge/instance-vflfzz0z with job batch system ID: 331 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:56:52-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpjrqxrkm2/worker_log.txt +[2020-11-26T14:56:52-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmphanqtimh/worker_log.txt +[2020-11-26T14:56:54-0800] [MainThread] [I] [toil.leader] Job ended: 'unmerge' kind-unmerge/instance-ritimmsk +[2020-11-26T14:56:54-0800] [MainThread] [I] [toil.leader] Job ended: 'unmerge' kind-unmerge/instance-vflfzz0z +[2020-11-26T14:56:54-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-47mtlvem with job batch system ID: 332 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-gjrkfl_h/chr1.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-sba51l1c/chr10.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-_1f9e0wl/chr2.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-odosh7jb/chr21.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-d1_hltjj/chr9.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-9rg4oku4/chr12.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-1o9lpmoy/chr15.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-s70n373i/chr11.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-wikgt8vr/chr17.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-nms5nl7h/chr4.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-9_0n5le7/chr13.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-yoa2em9h/chr3.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-erddr0j4/chr16.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-cuj8rw9n/chr5.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-h6zpvjpd/chr19.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-cojp7267/chr14.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-sa9q6wc4/chr7.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-b6agqmov/chrX.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-cd9f7_b6/chr8.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-q4f1ocun/chr6.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-svzx81i6/chr20.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-33_x36dv/chrM.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-3ok31_kh/chrY.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-9c4bqcrr/chr18.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-vflfzz0z/file-49o0zoux/chr22.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-jpycfr0z/chr3.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-mtwrl5br/chrM.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-81b_tgis/chr13.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-624hnrva/chr14.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-gqm077l0/chr19.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-xo31cu9n/chr4.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-qmr6qgsu/chr8.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-bi6b6fdh/chr7.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-cg25k6ps/chr11.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-hoxlamtw/chr5.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-se51jdf8/chrX.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-265io4w0/chr12.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-v1xzbo00/chr22.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-g75muhql/chr18.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-c2yjlxst/chr6.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-kjxjpidx/chr10.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-hw64q02y/chr2.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-j98ijv0u/chr20.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-_eik0ybg/chr1.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-wxbwqgk8/chrY.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-_4ei48wb/chr17.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-e_4x9myd/chr15.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-zsfg6oq_/chr9.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-dcq1rvl2/chr16.vcf to output location +[2020-11-26T14:56:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:56:54: Exporting files/for-job/kind-unmerge/instance-ritimmsk/file-zc8l61fs/chr21.vcf to output location +[2020-11-26T14:56:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpj1mzjv__/worker_log.txt +[2020-11-26T14:56:55-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-47mtlvem +[2020-11-26T14:56:55-0800] [MainThread] [I] [toil.leader] Issued job 'wrap_unmerge' kind-EncapsulatedJob/instance-7e2mu_4f with job batch system ID: 333 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:56:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpcgmfy_42/worker_log.txt +[2020-11-26T14:56:55-0800] [MainThread] [I] [toil.leader] Job ended: 'wrap_unmerge' kind-EncapsulatedJob/instance-7e2mu_4f +[2020-11-26T14:56:55-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-v67dee47 with job batch system ID: 334 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:56:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpqg9shpie/worker_log.txt +[2020-11-26T14:56:56-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-v67dee47 +[2020-11-26T14:56:56-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-6w2fs77j with job batch system ID: 335 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:56:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpztupil4r/worker_log.txt +[2020-11-26T14:56:57-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-6w2fs77j +[2020-11-26T14:57:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:50: Filtered SomaticSniper for TEST successfully +[2020-11-26T14:57:51-0800] [MainThread] [I] [toil.leader] Job ended: 'filter_somaticsniper' kind-filter_somaticsniper/instance-obid9nr_ +[2020-11-26T14:57:51-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-sk4fzcaz with job batch system ID: 336 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:57:52-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpsgo081yx/worker_log.txt +[2020-11-26T14:57:52-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-sk4fzcaz +[2020-11-26T14:57:52-0800] [MainThread] [I] [toil.leader] Issued job 'unmerge' kind-unmerge/instance-suu4_fcq with job batch system ID: 337 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:57:52-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp2wr8g70f/worker_log.txt +[2020-11-26T14:57:54-0800] [MainThread] [I] [toil.leader] Job ended: 'unmerge' kind-unmerge/instance-suu4_fcq +[2020-11-26T14:57:54-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-sk4fzcaz with job batch system ID: 338 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-v1fq10hn/chr4.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-j81tgvh3/chr6.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-ohr_satv/chr7.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-ck0dqnpk/chr8.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-oxp5zyy5/chr15.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-omq2hma8/chr16.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-ray_q47n/chrM.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-s1_7v_cb/chr5.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-go02tjqf/chr3.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-7e8cvjao/chrY.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-fqltqcgf/chrX.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-43ggrq_1/chr9.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-j_2bl0_9/chr19.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-bu8p7r7q/chr14.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-d_y1mi_5/chr18.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-yo_s9710/chr13.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-_n06ooma/chr17.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-fjcf7ttt/chr21.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-g0e0j7ui/chr12.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-cb6huyj4/chr2.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-johln4j8/chr20.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-whrnidf9/chr10.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-a_je388r/chr11.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-plqzh9_4/chr22.vcf to output location +[2020-11-26T14:57:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:57:55: Exporting files/for-job/kind-unmerge/instance-suu4_fcq/file-qmmuja69/chr1.vcf to output location +[2020-11-26T14:57:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp8985wj9n/worker_log.txt +[2020-11-26T14:57:55-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-sk4fzcaz +[2020-11-26T14:57:55-0800] [MainThread] [I] [toil.leader] Issued job 'filter_somaticsniper' kind-EncapsulatedJob/instance-7a1dtxge with job batch system ID: 339 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:57:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpqjw6y6ho/worker_log.txt +[2020-11-26T14:57:56-0800] [MainThread] [I] [toil.leader] Job ended: 'filter_somaticsniper' kind-EncapsulatedJob/instance-7a1dtxge +[2020-11-26T14:57:56-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-98y719jf with job batch system ID: 340 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:57:56-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-1se_o02s with job batch system ID: 341 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:57:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpuqjtrxzu/worker_log.txt +[2020-11-26T14:57:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpp2elef6x/worker_log.txt +[2020-11-26T14:57:57-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-98y719jf +[2020-11-26T14:57:57-0800] [MainThread] [I] [toil.leader] Issued job 'run_pileup' kind-EncapsulatedJob/instance-vbeka1dz with job batch system ID: 342 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:57:57-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-1se_o02s +[2020-11-26T14:57:57-0800] [MainThread] [I] [toil.leader] Issued job 'run_somaticsniper_full' kind-EncapsulatedJob/instance-cez1709k with job batch system ID: 343 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T14:57:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpolp4ipoz/worker_log.txt +[2020-11-26T14:57:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpw6_zya42/worker_log.txt +[2020-11-26T14:57:57-0800] [MainThread] [I] [toil.leader] Job ended: 'run_somaticsniper_full' kind-EncapsulatedJob/instance-cez1709k +[2020-11-26T14:57:58-0800] [MainThread] [I] [toil.leader] Job ended: 'run_pileup' kind-EncapsulatedJob/instance-vbeka1dz +[2020-11-26T14:57:58-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-prbhtump with job batch system ID: 344 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:57:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp22r5b3u3/worker_log.txt +[2020-11-26T14:57:59-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-prbhtump +[2020-11-26T14:59:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:59:16: Exporting files/for-job/kind-run_star/instance-ccd4mgp2/file-j208i6tm/rnaAligned.toTranscriptome.out.bam to output location +[2020-11-26T14:59:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:59:16: Exporting files/for-job/kind-run_star/instance-ccd4mgp2/file-q9eaiig2/rnaChimeric.out.junction to output location +[2020-11-26T14:59:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:59:16: Ran STAR on TEST successfully +[2020-11-26T14:59:16-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 14:59:16: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-9a1recg5/cleanup/file-q2fym03_/stream used 112.45% (60.3 GB [64780681216B] used, 53.7 GB [57610702127B] requested) at the end of its run. +[2020-11-26T14:59:24-0800] [MainThread] [I] [toil.leader] Job ended: 'run_star' kind-run_star/instance-ccd4mgp2 +[2020-11-26T14:59:24-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-x39gtorz with job batch system ID: 345 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T14:59:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpicink71r/worker_log.txt +[2020-11-26T14:59:25-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-x39gtorz +[2020-11-26T14:59:25-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-bvaaqr10 with job batch system ID: 346 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T14:59:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp274h3sg3/worker_log.txt +[2020-11-26T15:00:00-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-bvaaqr10 +[2020-11-26T15:00:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-x39gtorz with job batch system ID: 347 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:00:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:00:00: Ran samtools-sort on TEST:rna successfully +[2020-11-26T15:00:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:00:00: Exporting files/for-job/kind-EncapsulatedJob/instance-bvaaqr10/file-w7tefkwb/rna_sorted.bam to output location +[2020-11-26T15:00:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:00:00: Exporting files/for-job/kind-EncapsulatedJob/instance-bvaaqr10/file-3gz__o_u/rna_genome_sorted.bam.bai to output location +[2020-11-26T15:00:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:00:00: Ran samtools-index on TEST:rna successfully +[2020-11-26T15:00:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpg1e9vbxr/worker_log.txt +[2020-11-26T15:00:01-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-x39gtorz +[2020-11-26T15:00:01-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-qb0mgfur with job batch system ID: 348 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:00:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpvk1dnash/worker_log.txt +[2020-11-26T15:00:02-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-qb0mgfur +[2020-11-26T15:00:02-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-n3rrypxi with job batch system ID: 349 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-26T15:00:02-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-oicmm5uw with job batch system ID: 350 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-26T15:00:02-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-_6rphwl0 with job batch system ID: 351 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-26T15:00:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp1yx9wv5w/worker_log.txt +[2020-11-26T15:00:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmprx40lwr1/worker_log.txt +[2020-11-26T15:00:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpf371mu7a/worker_log.txt +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-oicmm5uw +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'run_rsem' kind-run_rsem/instance-foim_aa3 with job batch system ID: 352 and cores: 80, disk: 5.0 G, and memory: 2.0 G +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-n3rrypxi +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'run_fusion' kind-run_fusion/instance-npmmw7gn with job batch system ID: 353 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-_6rphwl0 +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-p911byyp with job batch system ID: 354 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-x1_n2p8i with job batch system ID: 355 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-_wmd1bp1 with job batch system ID: 356 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-mthp3vnn with job batch system ID: 357 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-lc_w67ku with job batch system ID: 358 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-12ecnzqr with job batch system ID: 359 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-s70jxiyq with job batch system ID: 360 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-y7tib8ab with job batch system ID: 361 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-doibjk8h with job batch system ID: 362 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-x0y18cwc with job batch system ID: 363 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-sc0yical with job batch system ID: 364 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-hymu_3o5 with job batch system ID: 365 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-ypn1choc with job batch system ID: 366 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-udws25tf with job batch system ID: 367 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-0ut4vu6v with job batch system ID: 368 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-vc7jncig with job batch system ID: 369 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-h5lqdtwn with job batch system ID: 370 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-xuqdac_1 with job batch system ID: 371 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-zuz3jaxj with job batch system ID: 372 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-9_x_gku_ with job batch system ID: 373 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-5k9ez0fu with job batch system ID: 374 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-zmtu9wwx with job batch system ID: 375 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-tfsul5yg with job batch system ID: 376 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-ve2xocb3 with job batch system ID: 377 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-8i4q6zgo with job batch system ID: 378 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:00:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpyt1wdfs3/worker_log.txt +[2020-11-26T15:00:04-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:00:04: Ran spawn_radia on TEST successfully +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpi9uw65hw/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpbobz6ghy/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp5mat_4f9/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp0z7do8ul/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp9_lfdssp/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpxx_4b1me/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpcixtj6n7/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpvqyoxefe/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpg_ek87u4/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpglkqyp7o/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp33psokny/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpe7k8c9f1/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmps_bjyb4e/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp2psarts8/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpbk215r3b/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpv3te931h/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpsb0kgmkf/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpgdeji2q2/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpyfjnftek/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp9g0ksyxf/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpy9whwtxi/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpzvx4f0n_/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpeyq4sahg/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp1vv3nrx5/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpcyi6b05d/worker_log.txt +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-p911byyp +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-shjnst4w with job batch system ID: 379 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-_wmd1bp1 +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-g9mnczpx with job batch system ID: 380 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-mthp3vnn +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-g4zrwcvu with job batch system ID: 381 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-12ecnzqr +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-v4ve9h41 with job batch system ID: 382 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-doibjk8h +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-zhn9alkw with job batch system ID: 383 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-x0y18cwc +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-tpcxgl4x with job batch system ID: 384 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-lc_w67ku +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-a2pjialu with job batch system ID: 385 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-x1_n2p8i +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-ey0a4uv2 with job batch system ID: 386 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-s70jxiyq +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-4klhf_wk with job batch system ID: 387 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-ypn1choc +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-swyr6x2g with job batch system ID: 388 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-vc7jncig +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-uwtw714t with job batch system ID: 389 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-h5lqdtwn +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-spp5cibv with job batch system ID: 390 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-y7tib8ab +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-64_l1okk with job batch system ID: 391 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-sc0yical +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-7qcm1iil with job batch system ID: 392 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-hymu_3o5 +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-k23xasj_ with job batch system ID: 393 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-udws25tf +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-wzfv7f28 with job batch system ID: 394 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-0ut4vu6v +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-kazyqb3w with job batch system ID: 395 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-xuqdac_1 +[2020-11-26T15:00:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-nzr_b3va with job batch system ID: 396 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:06-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-zuz3jaxj +[2020-11-26T15:00:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-a5dadtl1 with job batch system ID: 397 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:06-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-9_x_gku_ +[2020-11-26T15:00:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-21m1x59s with job batch system ID: 398 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:06-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-5k9ez0fu +[2020-11-26T15:00:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-0zo5unng with job batch system ID: 399 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:06-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-zmtu9wwx +[2020-11-26T15:00:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-ui_c4z_2 with job batch system ID: 400 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:06-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-tfsul5yg +[2020-11-26T15:00:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-32dgmc1x with job batch system ID: 401 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:06-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-ve2xocb3 +[2020-11-26T15:00:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-x4frh6l2 with job batch system ID: 402 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:06-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-8i4q6zgo +[2020-11-26T15:00:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-bdr6o0vr with job batch system ID: 403 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:00:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpt6av8alu/worker_log.txt +[2020-11-26T15:00:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmptu5xj8kp/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpoc7475o9/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmplya1tgzq/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpfkbczuo2/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpo53z8wvf/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp6oe0an44/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp55fe37ux/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpm00g49e7/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp3sey9xpz/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpg0bxv9ou/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpn740tnrd/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpamn3nbxl/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpd8jd6szj/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpxwe4nvhr/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpmt22mo70/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp25eayi8f/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmppr70t5xq/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpgeprl332/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpnvj1numj/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpkgdy9t4g/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp7hz7fkcq/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp_aey5yim/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmplctmzypq/worker_log.txt +[2020-11-26T15:00:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmps9i6fd3u/worker_log.txt +[2020-11-26T15:03:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:03:31: Ran radia on TEST:chrM successfully +[2020-11-26T15:03:32-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-bdr6o0vr +[2020-11-26T15:03:32-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-uk1z0kxa with job batch system ID: 404 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:03:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpnngs6f80/worker_log.txt +[2020-11-26T15:03:33-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-uk1z0kxa +[2020-11-26T15:03:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-4sndt0jb with job batch system ID: 405 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:03:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp95ggsxn3/worker_log.txt +[2020-11-26T15:05:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:05:14: Ran radia on TEST:chrY successfully +[2020-11-26T15:05:15-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-x4frh6l2 +[2020-11-26T15:05:15-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-lsajdqmf with job batch system ID: 406 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:05:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpfacaarl5/worker_log.txt +[2020-11-26T15:05:16-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-lsajdqmf +[2020-11-26T15:05:16-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-mrvzfj_o with job batch system ID: 407 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:05:17-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpxisgn5ek/worker_log.txt +[2020-11-26T15:06:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:06:26: Exporting files/for-job/kind-run_filter_radia/instance-mrvzfj_o/file-1td2a2cd/chrY.vcf to output location +[2020-11-26T15:06:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:06:26: Ran filter-radia on TEST:chrY successfully +[2020-11-26T15:06:26-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:06:26: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-lsajdqmf/cleanup/file-4r61i6cl/stream used 123.23% (6.5 GB [7020257280B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:06:27-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-mrvzfj_o +[2020-11-26T15:06:27-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-er17r73f with job batch system ID: 408 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:06:28-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp6rdqz_it/worker_log.txt +[2020-11-26T15:06:28-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-er17r73f +[2020-11-26T15:06:28-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-lsajdqmf with job batch system ID: 409 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:06:29-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpengvlypq/worker_log.txt +[2020-11-26T15:06:29-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-lsajdqmf +[2020-11-26T15:06:29-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-ve2xocb3 with job batch system ID: 410 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:06:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpsynf86qz/worker_log.txt +[2020-11-26T15:06:30-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-ve2xocb3 +[2020-11-26T15:06:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:06:55: Exporting files/for-job/kind-run_filter_radia/instance-4sndt0jb/file-elfd0v8i/chrM.vcf to output location +[2020-11-26T15:06:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:06:55: Ran filter-radia on TEST:chrM successfully +[2020-11-26T15:06:55-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:06:55: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-uk1z0kxa/cleanup/file-f4nvmbsk/stream used 123.22% (6.5 GB [7019675648B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:06:56-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-4sndt0jb +[2020-11-26T15:06:56-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-jgbhgcpg with job batch system ID: 411 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:06:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpvn_2u9sn/worker_log.txt +[2020-11-26T15:06:57-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-jgbhgcpg +[2020-11-26T15:06:57-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-uk1z0kxa with job batch system ID: 412 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:06:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpy_8owa86/worker_log.txt +[2020-11-26T15:06:58-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-uk1z0kxa +[2020-11-26T15:06:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-8i4q6zgo with job batch system ID: 413 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:06:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpaxxugp9j/worker_log.txt +[2020-11-26T15:06:59-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-8i4q6zgo +[2020-11-26T15:07:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:07:09: Exporting files/for-job/kind-run_rsem/instance-foim_aa3/file-ph3y9va1/rsem.genes.results to output location +[2020-11-26T15:07:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:07:09: Exporting files/for-job/kind-run_rsem/instance-foim_aa3/file-sxhdcxx8/rsem.isoforms.results to output location +[2020-11-26T15:07:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:07:09: Ran rsem on TEST successfully +[2020-11-26T15:07:10-0800] [MainThread] [I] [toil.leader] Job ended: 'run_rsem' kind-run_rsem/instance-foim_aa3 +[2020-11-26T15:07:10-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-p_yvadzn with job batch system ID: 414 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:07:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp5m8cwybw/worker_log.txt +[2020-11-26T15:07:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpo9um8ygp/worker_log.txt +[2020-11-26T15:07:11-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-p_yvadzn +[2020-11-26T15:07:11-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-4as6ip3q with job batch system ID: 415 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:07:12-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpgr1xv7kp/worker_log.txt +[2020-11-26T15:07:12-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-4as6ip3q +[2020-11-26T15:07:12-0800] [MainThread] [I] [toil.leader] Issued job 'run_mhc_gene_assessment' kind-run_mhc_gene_assessment/instance-sib6sb0p with job batch system ID: 416 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-26T15:07:12-0800] [MainThread] [I] [toil.leader] Issued job 'run_itx_resistance_assessment' kind-run_itx_resistance_assessment/instance-v2y_yxs8 with job batch system ID: 417 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-26T15:07:12-0800] [MainThread] [I] [toil.leader] Issued job 'run_car_t_validity_assessment' kind-run_car_t_validity_assessment/instance-hgko24n3 with job batch system ID: 418 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-26T15:07:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpxlpx04gx/worker_log.txt +[2020-11-26T15:07:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpqbpxhv6k/worker_log.txt +[2020-11-26T15:07:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpo6yymmfc/worker_log.txt +[2020-11-26T15:07:14-0800] [MainThread] [I] [toil.leader] Job ended: 'run_itx_resistance_assessment' kind-run_itx_resistance_assessment/instance-v2y_yxs8 +[2020-11-26T15:07:14-0800] [MainThread] [I] [toil.leader] Issued job 'assess_itx_resistance' kind-assess_itx_resistance/instance-eehwruic with job batch system ID: 419 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T15:07:14-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mhc_gene_assessment' kind-run_mhc_gene_assessment/instance-sib6sb0p +[2020-11-26T15:07:14-0800] [MainThread] [I] [toil.leader] Issued job 'assess_mhc_genes' kind-assess_mhc_genes/instance-31c8uucw with job batch system ID: 420 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T15:07:14-0800] [MainThread] [I] [toil.leader] Job ended: 'run_car_t_validity_assessment' kind-run_car_t_validity_assessment/instance-hgko24n3 +[2020-11-26T15:07:14-0800] [MainThread] [I] [toil.leader] Issued job 'assess_car_t_validity' kind-assess_car_t_validity/instance-uplh1mku with job batch system ID: 421 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-26T15:07:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpvsvmtvca/worker_log.txt +[2020-11-26T15:07:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpwjazjf8b/worker_log.txt +[2020-11-26T15:07:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpqu5f00nv/worker_log.txt +[2020-11-26T15:07:15-0800] [MainThread] [I] [toil.leader] Job ended: 'assess_itx_resistance' kind-assess_itx_resistance/instance-eehwruic +[2020-11-26T15:07:15-0800] [MainThread] [I] [toil.leader] Issued job 'run_itx_resistance_assessment' kind-run_itx_resistance_assessment/instance-v2y_yxs8 with job batch system ID: 422 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-26T15:07:15-0800] [MainThread] [I] [toil.leader] Job ended: 'assess_car_t_validity' kind-assess_car_t_validity/instance-uplh1mku +[2020-11-26T15:07:15-0800] [MainThread] [I] [toil.leader] Issued job 'run_car_t_validity_assessment' kind-run_car_t_validity_assessment/instance-hgko24n3 with job batch system ID: 423 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-26T15:07:15-0800] [MainThread] [I] [toil.leader] Job ended: 'assess_mhc_genes' kind-assess_mhc_genes/instance-31c8uucw +[2020-11-26T15:07:15-0800] [MainThread] [I] [toil.leader] Issued job 'run_mhc_gene_assessment' kind-run_mhc_gene_assessment/instance-sib6sb0p with job batch system ID: 424 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-26T15:07:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:07:16: Exporting files/for-job/kind-assess_car_t_validity/instance-uplh1mku/file-s1yr58pd/car_t_target_report.txt to output location +[2020-11-26T15:07:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:07:16: Ran car t validity assessment on TEST successfully +[2020-11-26T15:07:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:07:16: Exporting files/for-job/kind-assess_mhc_genes/instance-31c8uucw/file-khvdixq0/mhc_pathway_report.txt to output location +[2020-11-26T15:07:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:07:16: Ran mhc gene assessment on TEST successfully +[2020-11-26T15:07:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:07:16: Exporting files/for-job/kind-assess_itx_resistance/instance-eehwruic/file-2wdge3an/immunotherapy_resistance_report.txt to output location +[2020-11-26T15:07:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:07:16: Ran create immunotherapy resistance report on TEST successfully +[2020-11-26T15:07:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpn111sgd3/worker_log.txt +[2020-11-26T15:07:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp4u1ez_tq/worker_log.txt +[2020-11-26T15:07:16-0800] [MainThread] [I] [toil.leader] Job ended: 'run_car_t_validity_assessment' kind-run_car_t_validity_assessment/instance-hgko24n3 +[2020-11-26T15:07:16-0800] [MainThread] [I] [toil.leader] Job ended: 'run_itx_resistance_assessment' kind-run_itx_resistance_assessment/instance-v2y_yxs8 +[2020-11-26T15:07:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpokhlzavi/worker_log.txt +[2020-11-26T15:07:17-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mhc_gene_assessment' kind-run_mhc_gene_assessment/instance-sib6sb0p +[2020-11-26T15:10:29-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:10:29: Ran radia on TEST:chr21 successfully +[2020-11-26T15:10:29-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-0zo5unng +[2020-11-26T15:10:29-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-avkh_sll with job batch system ID: 425 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:10:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpm7qxu6ay/worker_log.txt +[2020-11-26T15:10:30-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-avkh_sll +[2020-11-26T15:10:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-rm7v1ze4 with job batch system ID: 426 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:10:31-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp7lmdaymf/worker_log.txt +[2020-11-26T15:11:08-0800] [MainThread] [I] [toil.leader] 49 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T15:11:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:11:46: Exporting files/for-job/kind-run_filter_radia/instance-rm7v1ze4/file-5t5m1oh4/chr21.vcf to output location +[2020-11-26T15:11:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:11:46: Ran filter-radia on TEST:chr21 successfully +[2020-11-26T15:11:46-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:11:46: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-avkh_sll/cleanup/file-lqqhhlqm/stream used 123.18% (6.5 GB [7017537536B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:11:47-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-rm7v1ze4 +[2020-11-26T15:11:47-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-iu7j0df1 with job batch system ID: 427 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:11:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpcdjb9nrm/worker_log.txt +[2020-11-26T15:11:56-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-iu7j0df1 +[2020-11-26T15:11:56-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-avkh_sll with job batch system ID: 428 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:11:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpqj5mz6hn/worker_log.txt +[2020-11-26T15:11:57-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-avkh_sll +[2020-11-26T15:11:57-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-5k9ez0fu with job batch system ID: 429 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:11:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp8c0gfypl/worker_log.txt +[2020-11-26T15:11:57-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-5k9ez0fu +[2020-11-26T15:13:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:13:15: Ran radia on TEST:chr22 successfully +[2020-11-26T15:13:16-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-ui_c4z_2 +[2020-11-26T15:13:16-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-g80394xg with job batch system ID: 430 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:13:17-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpx72_lx1v/worker_log.txt +[2020-11-26T15:13:17-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-g80394xg +[2020-11-26T15:13:17-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-k0c8lnz_ with job batch system ID: 431 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:13:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpjov2g4mc/worker_log.txt +[2020-11-26T15:14:36-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:14:36: Exporting files/for-job/kind-run_filter_radia/instance-k0c8lnz_/file-5repnsqg/chr22.vcf to output location +[2020-11-26T15:14:36-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:14:36: Ran filter-radia on TEST:chr22 successfully +[2020-11-26T15:14:36-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:14:36: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-g80394xg/cleanup/file-sidoz3xt/stream used 123.19% (6.5 GB [7018123264B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:14:38-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-k0c8lnz_ +[2020-11-26T15:14:38-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-xlmwq15l with job batch system ID: 432 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:14:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpbnl0f6wu/worker_log.txt +[2020-11-26T15:14:39-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-xlmwq15l +[2020-11-26T15:14:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-g80394xg with job batch system ID: 433 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:14:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpn19a5j0r/worker_log.txt +[2020-11-26T15:14:40-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-g80394xg +[2020-11-26T15:14:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-zmtu9wwx with job batch system ID: 434 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:14:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp54_rlwjp/worker_log.txt +[2020-11-26T15:14:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-zmtu9wwx +[2020-11-26T15:15:53-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:15:53: Ran radia on TEST:chr19 successfully +[2020-11-26T15:15:54-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-a5dadtl1 +[2020-11-26T15:15:54-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-cq4yxita with job batch system ID: 435 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:15:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpq2hexliu/worker_log.txt +[2020-11-26T15:15:55-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-cq4yxita +[2020-11-26T15:15:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-8bxllzt9 with job batch system ID: 436 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:15:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmphlnzdy5s/worker_log.txt +[2020-11-26T15:17:05-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:17:05: Ran radia on TEST:chr20 successfully +[2020-11-26T15:17:06-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-21m1x59s +[2020-11-26T15:17:06-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-4u6lu39s with job batch system ID: 437 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:17:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpxon4tx22/worker_log.txt +[2020-11-26T15:17:07-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-4u6lu39s +[2020-11-26T15:17:07-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-ogbbflkp with job batch system ID: 438 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:17:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpoefkmsls/worker_log.txt +[2020-11-26T15:17:22-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:17:22: Exporting files/for-job/kind-run_filter_radia/instance-8bxllzt9/file-kxtmd9w8/chr19.vcf to output location +[2020-11-26T15:17:22-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:17:22: Ran filter-radia on TEST:chr19 successfully +[2020-11-26T15:17:22-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:17:22: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-cq4yxita/cleanup/file-ovbp20s9/stream used 123.21% (6.5 GB [7019335680B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:17:24-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-8bxllzt9 +[2020-11-26T15:17:24-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ls4ex9hi with job batch system ID: 439 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:17:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpgpz3b959/worker_log.txt +[2020-11-26T15:17:24-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ls4ex9hi +[2020-11-26T15:17:24-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-cq4yxita with job batch system ID: 440 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:17:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpsor10b1d/worker_log.txt +[2020-11-26T15:17:25-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-cq4yxita +[2020-11-26T15:17:25-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-zuz3jaxj with job batch system ID: 441 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:17:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp2l1h6ogc/worker_log.txt +[2020-11-26T15:17:26-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-zuz3jaxj +[2020-11-26T15:18:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:18:42: Exporting files/for-job/kind-run_filter_radia/instance-ogbbflkp/file-98j9k83k/chr20.vcf to output location +[2020-11-26T15:18:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:18:42: Ran filter-radia on TEST:chr20 successfully +[2020-11-26T15:18:42-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:18:42: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-4u6lu39s/cleanup/file-gzbj4ip1/stream used 123.20% (6.5 GB [7018471424B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:18:43-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-ogbbflkp +[2020-11-26T15:18:43-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-1vb_4cl0 with job batch system ID: 442 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:18:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpkr1mld_t/worker_log.txt +[2020-11-26T15:18:44-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-1vb_4cl0 +[2020-11-26T15:18:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-4u6lu39s with job batch system ID: 443 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:18:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp_79qfdbz/worker_log.txt +[2020-11-26T15:18:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-4u6lu39s +[2020-11-26T15:18:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-9_x_gku_ with job batch system ID: 444 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:18:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpuvzcwo0_/worker_log.txt +[2020-11-26T15:18:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-9_x_gku_ +[2020-11-26T15:19:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:19:37: Ran radia on TEST:chr18 successfully +[2020-11-26T15:19:37-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-nzr_b3va +[2020-11-26T15:19:37-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-pegz4guq with job batch system ID: 445 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:19:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmprnggd0wd/worker_log.txt +[2020-11-26T15:19:38-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-pegz4guq +[2020-11-26T15:19:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-5aqyl68j with job batch system ID: 446 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:19:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpgyuw2a4o/worker_log.txt +[2020-11-26T15:20:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:20:55: Ran radia on TEST:chr17 successfully +[2020-11-26T15:20:55-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-spp5cibv +[2020-11-26T15:20:55-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-pw8ipuz6 with job batch system ID: 447 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:20:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpzfa7z2no/worker_log.txt +[2020-11-26T15:20:56-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-pw8ipuz6 +[2020-11-26T15:20:56-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-kyw3vk1y with job batch system ID: 448 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:20:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:20:57: Ran radia on TEST:chr16 successfully +[2020-11-26T15:20:57-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-uwtw714t +[2020-11-26T15:20:57-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-v5dclely with job batch system ID: 449 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:20:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmplub1lpr6/worker_log.txt +[2020-11-26T15:20:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpuc2hkhf_/worker_log.txt +[2020-11-26T15:20:58-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-v5dclely +[2020-11-26T15:20:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-azxnqmah with job batch system ID: 450 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:20:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpm8b2u995/worker_log.txt +[2020-11-26T15:21:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:21:01: Exporting files/for-job/kind-run_filter_radia/instance-5aqyl68j/file-zqnw6qn9/chr18.vcf to output location +[2020-11-26T15:21:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:21:01: Ran filter-radia on TEST:chr18 successfully +[2020-11-26T15:21:01-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:21:01: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-pegz4guq/cleanup/file-rl4ub04f/stream used 123.21% (6.5 GB [7019204608B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:21:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-5aqyl68j +[2020-11-26T15:21:02-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-speojc67 with job batch system ID: 451 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:21:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpcvy90z_4/worker_log.txt +[2020-11-26T15:21:02-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-speojc67 +[2020-11-26T15:21:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-pegz4guq with job batch system ID: 452 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:21:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpe4_cmvoi/worker_log.txt +[2020-11-26T15:21:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-pegz4guq +[2020-11-26T15:21:03-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-xuqdac_1 with job batch system ID: 453 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:21:04-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp2yl7ths0/worker_log.txt +[2020-11-26T15:21:04-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-xuqdac_1 +[2020-11-26T15:21:36-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:21:36: Ran radia on TEST:chr13 successfully +[2020-11-26T15:21:37-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-swyr6x2g +[2020-11-26T15:21:37-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-6x60bld2 with job batch system ID: 454 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:21:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpuy28qay2/worker_log.txt +[2020-11-26T15:21:41-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-6x60bld2 +[2020-11-26T15:21:41-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-q39j2es2 with job batch system ID: 455 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:21:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpto02grlg/worker_log.txt +[2020-11-26T15:21:47-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:21:47: Ran radia on TEST:chr15 successfully +[2020-11-26T15:21:47-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-kazyqb3w +[2020-11-26T15:21:47-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-6i4bamkz with job batch system ID: 456 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:21:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmprrr9ncc6/worker_log.txt +[2020-11-26T15:21:48-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-6i4bamkz +[2020-11-26T15:21:48-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-o5p7i1ko with job batch system ID: 457 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:21:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpdocsqtq8/worker_log.txt +[2020-11-26T15:22:06-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:22:06: Ran radia on TEST:chrX successfully +[2020-11-26T15:22:06-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-32dgmc1x +[2020-11-26T15:22:06-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-s3adip59 with job batch system ID: 458 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:22:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpp1_gcu3p/worker_log.txt +[2020-11-26T15:22:11-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-s3adip59 +[2020-11-26T15:22:11-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-kujeurwr with job batch system ID: 459 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:22:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp6vfhgddn/worker_log.txt +[2020-11-26T15:22:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:22:28: Exporting files/for-job/kind-run_filter_radia/instance-kyw3vk1y/file-_d36ud37/chr17.vcf to output location +[2020-11-26T15:22:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:22:28: Ran filter-radia on TEST:chr17 successfully +[2020-11-26T15:22:28-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:22:28: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-pw8ipuz6/cleanup/file-s1l91870/stream used 123.23% (6.5 GB [7020040192B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:22:29-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-kyw3vk1y +[2020-11-26T15:22:29-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-50g2ooef with job batch system ID: 460 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:22:30-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:22:30: Exporting files/for-job/kind-run_filter_radia/instance-azxnqmah/file-sev0g6gt/chr16.vcf to output location +[2020-11-26T15:22:30-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:22:30: Ran filter-radia on TEST:chr16 successfully +[2020-11-26T15:22:30-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:22:30: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-v5dclely/cleanup/file-wnbjnios/stream used 123.24% (6.5 GB [7020969984B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:22:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpwgatosy9/worker_log.txt +[2020-11-26T15:22:30-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-50g2ooef +[2020-11-26T15:22:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-pw8ipuz6 with job batch system ID: 461 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:22:31-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-azxnqmah +[2020-11-26T15:22:31-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-a000ibzu with job batch system ID: 462 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:22:31-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpm64t2hjp/worker_log.txt +[2020-11-26T15:22:31-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-pw8ipuz6 +[2020-11-26T15:22:31-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-h5lqdtwn with job batch system ID: 463 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:22:31-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpo4h4gila/worker_log.txt +[2020-11-26T15:22:31-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-a000ibzu +[2020-11-26T15:22:31-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-v5dclely with job batch system ID: 464 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:22:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp8rlmdmch/worker_log.txt +[2020-11-26T15:22:32-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-h5lqdtwn +[2020-11-26T15:22:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp760yzwvr/worker_log.txt +[2020-11-26T15:22:32-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-v5dclely +[2020-11-26T15:22:32-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-vc7jncig with job batch system ID: 465 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:22:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpj8yw1t5e/worker_log.txt +[2020-11-26T15:22:33-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-vc7jncig +[2020-11-26T15:23:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:23:03: Exporting files/for-job/kind-run_filter_radia/instance-q39j2es2/file-hl1__b1h/chr13.vcf to output location +[2020-11-26T15:23:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:23:03: Ran filter-radia on TEST:chr13 successfully +[2020-11-26T15:23:03-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:23:03: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-6x60bld2/cleanup/file-wykzmc46/stream used 123.19% (6.5 GB [7017963520B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:23:04-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-q39j2es2 +[2020-11-26T15:23:04-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ii5egh9g with job batch system ID: 466 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:23:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpc9w8v1q0/worker_log.txt +[2020-11-26T15:23:05-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ii5egh9g +[2020-11-26T15:23:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-6x60bld2 with job batch system ID: 467 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:23:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpcw69wmqv/worker_log.txt +[2020-11-26T15:23:06-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-6x60bld2 +[2020-11-26T15:23:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-ypn1choc with job batch system ID: 468 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:23:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpbbpts2i6/worker_log.txt +[2020-11-26T15:23:07-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-ypn1choc +[2020-11-26T15:23:23-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:23:23: Exporting files/for-job/kind-run_filter_radia/instance-o5p7i1ko/file-zwkb28g6/chr15.vcf to output location +[2020-11-26T15:23:23-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:23:23: Ran filter-radia on TEST:chr15 successfully +[2020-11-26T15:23:23-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:23:23: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-6i4bamkz/cleanup/file-9v4t24vm/stream used 123.19% (6.5 GB [7018205184B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:23:24-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-o5p7i1ko +[2020-11-26T15:23:24-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-dn_ekagr with job batch system ID: 469 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:23:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpw28mtoca/worker_log.txt +[2020-11-26T15:23:25-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-dn_ekagr +[2020-11-26T15:23:25-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-6i4bamkz with job batch system ID: 470 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:23:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmppj19a37c/worker_log.txt +[2020-11-26T15:23:26-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-6i4bamkz +[2020-11-26T15:23:26-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-0ut4vu6v with job batch system ID: 471 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:23:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpofa5qj_7/worker_log.txt +[2020-11-26T15:23:27-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-0ut4vu6v +[2020-11-26T15:23:36-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:23:36: Exporting files/for-job/kind-run_filter_radia/instance-kujeurwr/file-8d37hg55/chrX.vcf to output location +[2020-11-26T15:23:36-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:23:36: Ran filter-radia on TEST:chrX successfully +[2020-11-26T15:23:36-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:23:36: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-s3adip59/cleanup/file-sem6o8t6/stream used 123.20% (6.5 GB [7018270720B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:23:37-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-kujeurwr +[2020-11-26T15:23:37-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-j_ilrz8x with job batch system ID: 472 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:23:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp0hg59too/worker_log.txt +[2020-11-26T15:23:38-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-j_ilrz8x +[2020-11-26T15:23:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-s3adip59 with job batch system ID: 473 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:23:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp6qiqzs58/worker_log.txt +[2020-11-26T15:23:39-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-s3adip59 +[2020-11-26T15:23:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-tfsul5yg with job batch system ID: 474 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:23:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp8llhj9d7/worker_log.txt +[2020-11-26T15:23:40-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-tfsul5yg +[2020-11-26T15:24:32-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:24:32: Ran radia on TEST:chr14 successfully +[2020-11-26T15:24:33-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-wzfv7f28 +[2020-11-26T15:24:33-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-n8bcshh3 with job batch system ID: 475 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:24:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpjrehixme/worker_log.txt +[2020-11-26T15:24:34-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-n8bcshh3 +[2020-11-26T15:24:34-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-7g4sdwtt with job batch system ID: 476 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:24:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpc8tuw033/worker_log.txt +[2020-11-26T15:26:04-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:26:04: Exporting files/for-job/kind-run_filter_radia/instance-7g4sdwtt/file-h0sq3v7v/chr14.vcf to output location +[2020-11-26T15:26:04-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:26:04: Ran filter-radia on TEST:chr14 successfully +[2020-11-26T15:26:04-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:26:04: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-n8bcshh3/cleanup/file-8hf1qlih/stream used 123.18% (6.5 GB [7017529344B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:26:05-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-7g4sdwtt +[2020-11-26T15:26:05-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-0uyrva2y with job batch system ID: 477 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:26:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpkbqwsu8q/worker_log.txt +[2020-11-26T15:26:06-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-0uyrva2y +[2020-11-26T15:26:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-n8bcshh3 with job batch system ID: 478 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:26:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmph1wvovxm/worker_log.txt +[2020-11-26T15:26:07-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-n8bcshh3 +[2020-11-26T15:26:07-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-udws25tf with job batch system ID: 479 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:26:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpnlcg7ooa/worker_log.txt +[2020-11-26T15:26:07-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-udws25tf +[2020-11-26T15:27:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:27:31: Ran radia on TEST:chr9 successfully +[2020-11-26T15:27:32-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-zhn9alkw +[2020-11-26T15:27:32-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-9rjuh2i_ with job batch system ID: 480 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:27:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp1woduq_i/worker_log.txt +[2020-11-26T15:27:34-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-9rjuh2i_ +[2020-11-26T15:27:34-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-enju0npu with job batch system ID: 481 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:27:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpw7jsw0o8/worker_log.txt +[2020-11-26T15:29:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:29:03: Ran radia on TEST:chr11 successfully +[2020-11-26T15:29:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-7qcm1iil +[2020-11-26T15:29:03-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-lsrl0_s0 with job batch system ID: 482 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:29:04-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpl07g93a2/worker_log.txt +[2020-11-26T15:29:04-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-lsrl0_s0 +[2020-11-26T15:29:04-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-3_7fdww9 with job batch system ID: 483 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:29:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpt8bwbc4h/worker_log.txt +[2020-11-26T15:29:06-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:29:06: Exporting files/for-job/kind-run_filter_radia/instance-enju0npu/file-i7zu2l3s/chr9.vcf to output location +[2020-11-26T15:29:06-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:29:06: Ran filter-radia on TEST:chr9 successfully +[2020-11-26T15:29:06-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:29:06: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-9rjuh2i_/cleanup/file-xek86dvz/stream used 123.19% (6.5 GB [7017963520B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:29:08-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-enju0npu +[2020-11-26T15:29:08-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-xszdr2r5 with job batch system ID: 484 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:29:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp8j6gmoqq/worker_log.txt +[2020-11-26T15:29:08-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-xszdr2r5 +[2020-11-26T15:29:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-9rjuh2i_ with job batch system ID: 485 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:29:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpmlxop635/worker_log.txt +[2020-11-26T15:29:09-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-9rjuh2i_ +[2020-11-26T15:29:09-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-doibjk8h with job batch system ID: 486 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:29:10-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpsnyazmdh/worker_log.txt +[2020-11-26T15:29:10-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-doibjk8h +[2020-11-26T15:29:23-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:29:23: Ran radia on TEST:chr12 successfully +[2020-11-26T15:29:24-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-k23xasj_ +[2020-11-26T15:29:24-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-35j0yenv with job batch system ID: 487 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:29:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpjm_5f84k/worker_log.txt +[2020-11-26T15:29:25-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-35j0yenv +[2020-11-26T15:29:25-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-uxpm963f with job batch system ID: 488 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:29:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpr8piwm2k/worker_log.txt +[2020-11-26T15:29:53-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:29:53: Ran radia on TEST:chr10 successfully +[2020-11-26T15:29:54-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-tpcxgl4x +[2020-11-26T15:29:54-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-i17atytv with job batch system ID: 489 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:29:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpc6d0_tbq/worker_log.txt +[2020-11-26T15:29:55-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-i17atytv +[2020-11-26T15:29:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-31rejiz3 with job batch system ID: 490 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:29:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmptiza916o/worker_log.txt +[2020-11-26T15:30:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:30:45: Exporting files/for-job/kind-run_filter_radia/instance-3_7fdww9/file-n6db5oym/chr11.vcf to output location +[2020-11-26T15:30:45-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:30:45: Ran filter-radia on TEST:chr11 successfully +[2020-11-26T15:30:45-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:30:45: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-lsrl0_s0/cleanup/file-kd_wbdm6/stream used 123.22% (6.5 GB [7019872256B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:30:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-3_7fdww9 +[2020-11-26T15:30:46-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-nj5b5241 with job batch system ID: 491 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:30:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpo0o5qe2j/worker_log.txt +[2020-11-26T15:30:47-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-nj5b5241 +[2020-11-26T15:30:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-lsrl0_s0 with job batch system ID: 492 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:30:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp704shrcb/worker_log.txt +[2020-11-26T15:30:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-lsrl0_s0 +[2020-11-26T15:30:48-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-sc0yical with job batch system ID: 493 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:30:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpolei86y3/worker_log.txt +[2020-11-26T15:30:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-sc0yical +[2020-11-26T15:31:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:31:07: Exporting files/for-job/kind-run_filter_radia/instance-uxpm963f/file-zydq1lpf/chr12.vcf to output location +[2020-11-26T15:31:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:31:07: Ran filter-radia on TEST:chr12 successfully +[2020-11-26T15:31:07-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:31:07: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-35j0yenv/cleanup/file-vj1_qi9q/stream used 123.21% (6.5 GB [7019069440B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:31:08-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-uxpm963f +[2020-11-26T15:31:08-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-aikc6wsk with job batch system ID: 494 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:31:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpo3e99gk9/worker_log.txt +[2020-11-26T15:31:09-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-aikc6wsk +[2020-11-26T15:31:09-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-35j0yenv with job batch system ID: 495 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:31:10-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpdropbn0o/worker_log.txt +[2020-11-26T15:31:10-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-35j0yenv +[2020-11-26T15:31:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-hymu_3o5 with job batch system ID: 496 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:31:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmppmok_x0x/worker_log.txt +[2020-11-26T15:31:11-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-hymu_3o5 +[2020-11-26T15:31:34-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:31:34: Exporting files/for-job/kind-run_filter_radia/instance-31rejiz3/file-3lpmyans/chr10.vcf to output location +[2020-11-26T15:31:34-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:31:34: Ran filter-radia on TEST:chr10 successfully +[2020-11-26T15:31:34-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:31:34: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-i17atytv/cleanup/file-o62ftyt_/stream used 123.22% (6.5 GB [7019659264B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:31:35-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-31rejiz3 +[2020-11-26T15:31:35-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-q_7mbrex with job batch system ID: 497 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:31:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp65gdu696/worker_log.txt +[2020-11-26T15:31:36-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-q_7mbrex +[2020-11-26T15:31:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-i17atytv with job batch system ID: 498 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:31:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmplixhlszj/worker_log.txt +[2020-11-26T15:31:37-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-i17atytv +[2020-11-26T15:31:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-x0y18cwc with job batch system ID: 499 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:31:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp6bzvajwa/worker_log.txt +[2020-11-26T15:31:38-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-x0y18cwc +[2020-11-26T15:33:11-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:33:11: Ran radia on TEST:chr8 successfully +[2020-11-26T15:33:11-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-64_l1okk +[2020-11-26T15:33:11-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-0uy4dkvh with job batch system ID: 500 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:33:12-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp6qj1blsy/worker_log.txt +[2020-11-26T15:33:13-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-0uy4dkvh +[2020-11-26T15:33:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-2uw234c5 with job batch system ID: 501 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:33:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpc2f1jkmu/worker_log.txt +[2020-11-26T15:33:19-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:33:19: Exporting files/for-job/kind-run_fusion/instance-npmmw7gn/file-tpazmgpg/star-fusion.fusion_candidates.final.abridged to output location +[2020-11-26T15:33:19-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:33:19: Exporting files/for-job/kind-run_fusion/instance-npmmw7gn/file-t69px_p5/FusionInspector.fusion_predictions.final.abridged.FFPM to output location +[2020-11-26T15:33:19-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:33:19: Exporting files/for-job/kind-run_fusion/instance-npmmw7gn/file-ihqad84m/FusionInspector.gmap_trinity_GG.fusions.fasta to output location +[2020-11-26T15:33:19-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:33:19: Exporting files/for-job/kind-run_fusion/instance-npmmw7gn/file-tg1uzg0j/FusionInspector.gmap_trinity_GG.fusions.gff3 to output location +[2020-11-26T15:33:19-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:33:19: Exporting files/for-job/kind-run_fusion/instance-npmmw7gn/file-5h80u480/fusion.final to output location +[2020-11-26T15:33:19-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:33:19: Ran STAR-Fusion on TEST successfully +[2020-11-26T15:33:19-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:33:19: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-n3rrypxi/cleanup/file-2hjpgnkf/stream used 111.61% (59.6 GB [64029351936B] used, 53.4 GB [57369019908B] requested) at the end of its run. +[2020-11-26T15:33:19-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:33:19: Exporting files/for-job/kind-run_fusion/instance-npmmw7gn/file-5q8zoyd_/fusion_results.bedpe to output location +[2020-11-26T15:33:19-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:33:19: Reformatted STAR-Fusion output for TEST successfully +[2020-11-26T15:33:28-0800] [MainThread] [I] [toil.leader] Job ended: 'run_fusion' kind-run_fusion/instance-npmmw7gn +[2020-11-26T15:33:28-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-s8mpby_h with job batch system ID: 502 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:33:29-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpum70x_zl/worker_log.txt +[2020-11-26T15:33:29-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-s8mpby_h +[2020-11-26T15:33:29-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-3ihq501k with job batch system ID: 503 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:33:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp9humexb9/worker_log.txt +[2020-11-26T15:33:30-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-3ihq501k +[2020-11-26T15:33:30-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-3siirej0 with job batch system ID: 504 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:33:31-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp4af7a0gk/worker_log.txt +[2020-11-26T15:33:31-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-3siirej0 +[2020-11-26T15:33:31-0800] [MainThread] [I] [toil.leader] Issued job 'delete_fastqs' kind-delete_fastqs/instance-x3vtays5 with job batch system ID: 505 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-26T15:33:31-0800] [MainThread] [I] [toil.leader] Issued job 'delete_fastqs' kind-delete_fastqs/instance-m4rscscl with job batch system ID: 506 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-26T15:33:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpbnjvoo4x/worker_log.txt +[2020-11-26T15:33:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpn_nxryv6/worker_log.txt +[2020-11-26T15:33:32-0800] [MainThread] [I] [toil.leader] Job ended: 'delete_fastqs' kind-delete_fastqs/instance-m4rscscl +[2020-11-26T15:33:33-0800] [MainThread] [I] [toil.leader] Job ended: 'delete_fastqs' kind-delete_fastqs/instance-x3vtays5 +[2020-11-26T15:33:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:33:33: Deleting "TEST:tumor_rna_fastq_1" from the filestore. +[2020-11-26T15:33:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:33:33: Deleting "TEST:tumor_dna_fastq_2" from the filestore. +[2020-11-26T15:33:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:33:33: Deleting "TEST:tumor_dna_fastq_1" from the filestore. +[2020-11-26T15:33:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:33:33: Deleting "TEST:tumor_rna_fastq_2" from the filestore. +[2020-11-26T15:33:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:33:33: Deleting "TEST:normal_dna_fastq_1" from the filestore. +[2020-11-26T15:33:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:33:33: Deleting "TEST:normal_dna_fastq_2" from the filestore. +[2020-11-26T15:34:51-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:34:51: Exporting files/for-job/kind-run_filter_radia/instance-2uw234c5/file-4ekbiv2h/chr8.vcf to output location +[2020-11-26T15:34:51-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:34:51: Ran filter-radia on TEST:chr8 successfully +[2020-11-26T15:34:51-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:34:51: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-0uy4dkvh/cleanup/file-husl4f7j/stream used 123.20% (6.5 GB [7018586112B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:34:52-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-2uw234c5 +[2020-11-26T15:34:52-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-74qdgu91 with job batch system ID: 507 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:34:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpzx0tsa80/worker_log.txt +[2020-11-26T15:34:53-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-74qdgu91 +[2020-11-26T15:34:53-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-0uy4dkvh with job batch system ID: 508 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:34:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpse7xje4d/worker_log.txt +[2020-11-26T15:34:54-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-0uy4dkvh +[2020-11-26T15:34:54-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-y7tib8ab with job batch system ID: 509 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:34:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpyro6k1h9/worker_log.txt +[2020-11-26T15:34:54-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-y7tib8ab +[2020-11-26T15:35:11-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:35:11: Ran radia on TEST:chr6 successfully +[2020-11-26T15:35:12-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-v4ve9h41 +[2020-11-26T15:35:12-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-bbrc96ii with job batch system ID: 510 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:35:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpk5p5cvsm/worker_log.txt +[2020-11-26T15:35:13-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-bbrc96ii +[2020-11-26T15:35:13-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-a24swtsc with job batch system ID: 511 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:35:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpdm2md_9u/worker_log.txt +[2020-11-26T15:36:53-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:36:53: Exporting files/for-job/kind-run_filter_radia/instance-a24swtsc/file-62n_f84y/chr6.vcf to output location +[2020-11-26T15:36:53-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:36:53: Ran filter-radia on TEST:chr6 successfully +[2020-11-26T15:36:53-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:36:53: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-bbrc96ii/cleanup/file-qtp94kol/stream used 123.23% (6.5 GB [7019925504B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:36:54-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-a24swtsc +[2020-11-26T15:36:54-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-xo_pxndv with job batch system ID: 512 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:36:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpc_khte9z/worker_log.txt +[2020-11-26T15:36:55-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-xo_pxndv +[2020-11-26T15:36:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-bbrc96ii with job batch system ID: 513 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:36:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpje3qbv1y/worker_log.txt +[2020-11-26T15:36:56-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-bbrc96ii +[2020-11-26T15:36:56-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-12ecnzqr with job batch system ID: 514 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:36:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpgapwo_ff/worker_log.txt +[2020-11-26T15:36:57-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-12ecnzqr +[2020-11-26T15:38:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:38:17: Ran radia on TEST:chr5 successfully +[2020-11-26T15:38:17-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-a2pjialu +[2020-11-26T15:38:17-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-x_31w7a_ with job batch system ID: 515 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:38:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpfk9cw585/worker_log.txt +[2020-11-26T15:38:19-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-x_31w7a_ +[2020-11-26T15:38:19-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-5iko4l06 with job batch system ID: 516 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:38:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp19lvji2l/worker_log.txt +[2020-11-26T15:39:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:39:14: Ran radia on TEST:chr4 successfully +[2020-11-26T15:39:14-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-g4zrwcvu +[2020-11-26T15:39:14-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-1al1eqlx with job batch system ID: 517 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:39:15-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpz9dfsdps/worker_log.txt +[2020-11-26T15:39:15-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-1al1eqlx +[2020-11-26T15:39:15-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-9uncwygg with job batch system ID: 518 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:39:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp4b_evir1/worker_log.txt +[2020-11-26T15:40:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:40:03: Exporting files/for-job/kind-run_filter_radia/instance-5iko4l06/file-cai9gq5b/chr5.vcf to output location +[2020-11-26T15:40:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:40:03: Ran filter-radia on TEST:chr5 successfully +[2020-11-26T15:40:03-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:40:03: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-x_31w7a_/cleanup/file-jk4nxqbj/stream used 123.26% (6.5 GB [7021936640B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:40:04-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-5iko4l06 +[2020-11-26T15:40:04-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-5u1p2e0h with job batch system ID: 519 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:40:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpwszz5mj9/worker_log.txt +[2020-11-26T15:40:05-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-5u1p2e0h +[2020-11-26T15:40:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-x_31w7a_ with job batch system ID: 520 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:40:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpj0cva9gm/worker_log.txt +[2020-11-26T15:40:06-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-x_31w7a_ +[2020-11-26T15:40:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-lc_w67ku with job batch system ID: 521 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:40:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp61ioy08z/worker_log.txt +[2020-11-26T15:40:07-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-lc_w67ku +[2020-11-26T15:40:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:40:59: Exporting files/for-job/kind-run_filter_radia/instance-9uncwygg/file-on879viy/chr4.vcf to output location +[2020-11-26T15:40:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:40:59: Ran filter-radia on TEST:chr4 successfully +[2020-11-26T15:40:59-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:40:59: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-1al1eqlx/cleanup/file-in3hiuai/stream used 123.32% (6.5 GB [7025307648B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:41:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-9uncwygg +[2020-11-26T15:41:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-1ultcre6 with job batch system ID: 522 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:41:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpgf8_6gnb/worker_log.txt +[2020-11-26T15:41:01-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-1ultcre6 +[2020-11-26T15:41:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-1al1eqlx with job batch system ID: 523 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:41:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpnbl2w5so/worker_log.txt +[2020-11-26T15:41:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-1al1eqlx +[2020-11-26T15:41:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-mthp3vnn with job batch system ID: 524 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:41:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmptdce268u/worker_log.txt +[2020-11-26T15:41:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-mthp3vnn +[2020-11-26T15:42:19-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:42:19: Ran radia on TEST:chr3 successfully +[2020-11-26T15:42:20-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-g9mnczpx +[2020-11-26T15:42:20-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-chh9t9oa with job batch system ID: 525 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:42:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:42:20: Ran radia on TEST:chr7 successfully +[2020-11-26T15:42:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpqwgx0ort/worker_log.txt +[2020-11-26T15:42:21-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-chh9t9oa +[2020-11-26T15:42:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-5r8vzcih with job batch system ID: 526 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:42:21-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-4klhf_wk +[2020-11-26T15:42:21-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-f02m50he with job batch system ID: 527 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:42:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpalhzwebu/worker_log.txt +[2020-11-26T15:42:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpgfhe13nq/worker_log.txt +[2020-11-26T15:42:22-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-f02m50he +[2020-11-26T15:42:22-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-gxwx02js with job batch system ID: 528 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:42:23-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp4wc9p9xx/worker_log.txt +[2020-11-26T15:44:04-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:44:04: Exporting files/for-job/kind-run_filter_radia/instance-gxwx02js/file-5wayh0jt/chr7.vcf to output location +[2020-11-26T15:44:04-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:44:04: Ran filter-radia on TEST:chr7 successfully +[2020-11-26T15:44:04-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:44:04: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-f02m50he/cleanup/file-q45wyqtq/stream used 123.20% (6.5 GB [7018770432B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:44:05-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-gxwx02js +[2020-11-26T15:44:05-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-xiy7t9og with job batch system ID: 529 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:44:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpjuzl9uw9/worker_log.txt +[2020-11-26T15:44:06-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-xiy7t9og +[2020-11-26T15:44:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-f02m50he with job batch system ID: 530 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:44:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpbto3b4kq/worker_log.txt +[2020-11-26T15:44:06-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-f02m50he +[2020-11-26T15:44:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-s70jxiyq with job batch system ID: 531 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:44:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp3dj6xgux/worker_log.txt +[2020-11-26T15:44:07-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-s70jxiyq +[2020-11-26T15:44:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:44:17: Exporting files/for-job/kind-run_filter_radia/instance-5r8vzcih/file-g3o9bq6a/chr3.vcf to output location +[2020-11-26T15:44:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:44:17: Ran filter-radia on TEST:chr3 successfully +[2020-11-26T15:44:17-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:44:17: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-chh9t9oa/cleanup/file-6xkzlalk/stream used 123.23% (6.5 GB [7020027904B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:44:18-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-5r8vzcih +[2020-11-26T15:44:18-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-0ienvzw6 with job batch system ID: 532 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:44:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp_sxl3q1l/worker_log.txt +[2020-11-26T15:44:18-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-0ienvzw6 +[2020-11-26T15:44:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-chh9t9oa with job batch system ID: 533 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:44:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp94b999dr/worker_log.txt +[2020-11-26T15:44:19-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-chh9t9oa +[2020-11-26T15:44:19-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-_wmd1bp1 with job batch system ID: 534 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:44:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpwuz9jgnm/worker_log.txt +[2020-11-26T15:44:20-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-_wmd1bp1 +[2020-11-26T15:49:22-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:49:22: Ran radia on TEST:chr2 successfully +[2020-11-26T15:49:23-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-ey0a4uv2 +[2020-11-26T15:49:23-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-4i07xrk5 with job batch system ID: 535 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:49:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpf4c4d4lt/worker_log.txt +[2020-11-26T15:49:24-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-4i07xrk5 +[2020-11-26T15:49:24-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-w7jl2s5v with job batch system ID: 536 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:49:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp1ry4xcsg/worker_log.txt +[2020-11-26T15:50:13-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:50:13: Ran radia on TEST:chr1 successfully +[2020-11-26T15:50:14-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-shjnst4w +[2020-11-26T15:50:14-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-fgkozaxo with job batch system ID: 537 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:50:15-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpt9iy3txm/worker_log.txt +[2020-11-26T15:50:15-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-fgkozaxo +[2020-11-26T15:50:15-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-q5sfj7g9 with job batch system ID: 538 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-26T15:50:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp9uzj4xno/worker_log.txt +[2020-11-26T15:51:27-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:51:27: Exporting files/for-job/kind-run_filter_radia/instance-w7jl2s5v/file-97b6fp27/chr2.vcf to output location +[2020-11-26T15:51:27-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:51:27: Ran filter-radia on TEST:chr2 successfully +[2020-11-26T15:51:27-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:51:27: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-4i07xrk5/cleanup/file-ptqqvww5/stream used 123.24% (6.5 GB [7020883968B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:51:28-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-w7jl2s5v +[2020-11-26T15:51:28-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-l8o1ahf8 with job batch system ID: 539 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:51:28-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpe8tm5ery/worker_log.txt +[2020-11-26T15:51:28-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-l8o1ahf8 +[2020-11-26T15:51:28-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-4i07xrk5 with job batch system ID: 540 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:51:29-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpt_zu4m7v/worker_log.txt +[2020-11-26T15:51:29-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-4i07xrk5 +[2020-11-26T15:51:29-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-x1_n2p8i with job batch system ID: 541 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:51:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmpgh2pkthl/worker_log.txt +[2020-11-26T15:51:30-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-x1_n2p8i +[2020-11-26T15:52:13-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:52:13: Exporting files/for-job/kind-run_filter_radia/instance-q5sfj7g9/file-kcy7fv6y/chr1.vcf to output location +[2020-11-26T15:52:13-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:52:13: Ran filter-radia on TEST:chr1 successfully +[2020-11-26T15:52:13-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-26-2020 15:52:13: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-fgkozaxo/cleanup/file-k9e8itky/stream used 123.27% (6.5 GB [7022583808B] used, 5.3 GB [5696827930B] requested) at the end of its run. +[2020-11-26T15:52:14-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-q5sfj7g9 +[2020-11-26T15:52:14-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-h2fct844 with job batch system ID: 542 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:52:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp66o9kgzb/worker_log.txt +[2020-11-26T15:52:15-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-h2fct844 +[2020-11-26T15:52:15-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-fgkozaxo with job batch system ID: 543 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:52:15-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp3_gxszhb/worker_log.txt +[2020-11-26T15:52:15-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-fgkozaxo +[2020-11-26T15:52:15-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-p911byyp with job batch system ID: 544 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-26T15:52:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp2sf6x8jn/worker_log.txt +[2020-11-26T15:52:16-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-p911byyp +[2020-11-26T15:52:16-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-66oasase with job batch system ID: 545 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-26T15:52:17-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-b5556609-bb3a-4e18-b032-485ddcc441af-e8611b53-bdab-4598-acae-87977f99831e/tmp049eftfi/worker_log.txt +[2020-11-26T15:52:17-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-66oasase +[2020-11-26T16:11:09-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T17:11:10-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T18:11:11-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T19:11:11-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T20:11:12-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T21:11:13-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T22:11:14-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-26T23:11:14-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-27T00:11:15-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-27T01:11:16-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-27T02:11:17-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-27T03:11:18-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-27T04:11:18-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-27T05:11:19-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-27T06:11:20-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-27T07:11:21-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-27T08:11:22-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-27T09:11:23-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-27T10:11:24-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-27T11:11:24-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-27T12:11:25-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-27T13:11:26-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run diff --git a/errors/2020-11-27-14-10-34.txt b/errors/2020-11-27-14-10-34.txt new file mode 100644 index 00000000..852e87e4 --- /dev/null +++ b/errors/2020-11-27-14-10-34.txt @@ -0,0 +1,718 @@ +[2020-11-27T14:10:36-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-27T14:10:36-0800] [MainThread] [I] [toil.leader] Issued job 'parse_config_file' kind-parse_config_file/instance-utcdr1zk with job batch system ID: 0 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp22z41ydk/worker_log.txt +[2020-11-27T14:10:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:37: Parsing config file +[2020-11-27T14:10:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:37: Obtaining tool inputs +[2020-11-27T14:10:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:37: Obtained tool inputs +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Job ended: 'parse_config_file' kind-parse_config_file/instance-utcdr1zk +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] 0 jobs are running, 0 jobs are issued and waiting to run +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-q2e4hax4 with job batch system ID: 1 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-_pgqif1s with job batch system ID: 2 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-b460cxvc with job batch system ID: 3 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-hhvzwv61 with job batch system ID: 4 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-hg1ifco9 with job batch system ID: 5 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-dz3ee358 with job batch system ID: 6 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-m1wenzhg with job batch system ID: 7 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-92mgm79m with job batch system ID: 8 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-pvqugon6 with job batch system ID: 9 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-95ltkomo with job batch system ID: 10 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-e6vzn0bx with job batch system ID: 11 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-j6d2cs1z with job batch system ID: 12 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-84czw0m_ with job batch system ID: 13 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5b6cc686 with job batch system ID: 14 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ipecs42j with job batch system ID: 15 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5scs5d45 with job batch system ID: 16 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-xmfl9efj with job batch system ID: 17 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ify9dp_j with job batch system ID: 18 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-86k5hkoi with job batch system ID: 19 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-cst8ecqv with job batch system ID: 20 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-qjz8vfdj with job batch system ID: 21 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-s7o9pug0 with job batch system ID: 22 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-mz9mj14s with job batch system ID: 23 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5kyfgfzd with job batch system ID: 24 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-cmj7zg9i with job batch system ID: 25 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-g2l1f_9z with job batch system ID: 26 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-4u748z9c with job batch system ID: 27 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-vrlmorbv with job batch system ID: 28 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-w6l93h_s with job batch system ID: 29 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbudqgls7/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp87ns35u_/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpsn5_rce1/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpze70uyu4/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmprqwfsmxf/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbsc3ucae/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpx4iyoo8w/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp42c0abv9/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp9vz31pra/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpvtxofi5b/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpg2h52vz0/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpohdiwbzt/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp4qyd8fmn/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp8s42bw8i/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7pm6hvwu/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmprncg9203/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1ohtegow/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp4b7_ibcc/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1tbxifhj/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpawli98by/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp2wqi_yz8/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmppwl6k7km/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpw37o8usf/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpu24ty5l7/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgc0tkzwp/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpxdv3pbj3/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpx07e3uv5/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp3pj6jla8/worker_log.txt +[2020-11-27T14:10:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmprjf4qsxn/worker_log.txt +[2020-11-27T14:10:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-xmfl9efj +[2020-11-27T14:10:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:40: Obtaining file (strelka:config_file) to the file job store +[2020-11-27T14:10:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:40: Obtaining file (indexes:genome_fai) to the file job store +[2020-11-27T14:10:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:40: Obtaining file (indexes:genome_dict) to the file job store +[2020-11-27T14:10:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:40: Obtaining file (radia:gencode_beds) to the file job store +[2020-11-27T14:10:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:40: Obtaining file (indexes:dbsnp_tbi) to the file job store +[2020-11-27T14:10:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:40: Obtaining file (reports:itx_resistance_file) to the file job store +[2020-11-27T14:10:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:40: Obtaining file (mhcii:method_file) to the file job store +[2020-11-27T14:10:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:40: Obtaining file (indexes:cosmic_idx) to the file job store +[2020-11-27T14:10:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:40: Obtaining file (reports:mhc_pathways_file) to the file job store +[2020-11-27T14:10:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-hg1ifco9 +[2020-11-27T14:10:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-dz3ee358 +[2020-11-27T14:10:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5scs5d45 +[2020-11-27T14:10:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-92mgm79m +[2020-11-27T14:10:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-e6vzn0bx +[2020-11-27T14:10:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-vrlmorbv +[2020-11-27T14:10:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-cmj7zg9i +[2020-11-27T14:10:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-g2l1f_9z +[2020-11-27T14:10:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-j6d2cs1z +[2020-11-27T14:10:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-4u748z9c +[2020-11-27T14:10:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-w6l93h_s +[2020-11-27T14:10:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-cst8ecqv +[2020-11-27T14:10:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ipecs42j +[2020-11-27T14:10:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-qjz8vfdj +[2020-11-27T14:10:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5kyfgfzd +[2020-11-27T14:10:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-5b6cc686 +[2020-11-27T14:10:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-m1wenzhg +[2020-11-27T14:10:40-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-95ltkomo +[2020-11-27T14:10:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:40: Obtaining file (mhci:method_file) to the file job store +[2020-11-27T14:10:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:40: Obtaining file (radia:cosmic_beds) to the file job store +[2020-11-27T14:10:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:40: Obtaining file (reports:immune_resistance_pathways_file) to the file job store +[2020-11-27T14:10:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:40: Obtaining file (transgene:gencode_transcript_fasta) to the file job store +[2020-11-27T14:10:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:40: Obtaining file (reports:car_t_targets_file) to the file job store +[2020-11-27T14:10:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:40: Obtaining file (indexes:dbsnp_idx) to the file job store +[2020-11-27T14:10:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:40: Obtaining file (transgene:gencode_annotation_gtf) to the file job store +[2020-11-27T14:10:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:40: Obtaining file (radia:pseudogene_beds) to the file job store +[2020-11-27T14:10:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:40: Obtaining file (radia:retrogene_beds) to the file job store +[2020-11-27T14:10:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:40: Obtaining file (indexes:cosmic_vcf) to the file job store +[2020-11-27T14:10:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:41: Obtaining file (transgene:gencode_peptide_fasta) to the file job store +[2020-11-27T14:10:41-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-86k5hkoi +[2020-11-27T14:10:48-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-hhvzwv61 +[2020-11-27T14:10:48-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-s7o9pug0 +[2020-11-27T14:10:49-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:49: Obtaining file (indexes:genome_fasta) to the file job store +[2020-11-27T14:10:49-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:49: Obtaining file (transgene:genome_fasta) to the file job store +[2020-11-27T14:10:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:57: Obtaining file (radia:dbsnp_beds) to the file job store +[2020-11-27T14:10:57-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-84czw0m_ +[2020-11-27T14:10:58-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-b460cxvc +[2020-11-27T14:10:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:10:59: Obtaining file (rsem:index) to the file job store +[2020-11-27T14:11:05-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-ify9dp_j +[2020-11-27T14:11:05-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:11:05: Obtaining file (snpeff:index) to the file job store +[2020-11-27T14:11:37-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-pvqugon6 +[2020-11-27T14:11:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:11:37: Obtaining file (indexes:dbsnp_vcf) to the file job store +[2020-11-27T14:11:53-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:11:53: Obtaining file (bwa:index) to the file job store +[2020-11-27T14:11:53-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-_pgqif1s +[2020-11-27T14:12:08-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-mz9mj14s +[2020-11-27T14:12:08-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:12:08: Obtaining file (phlat:index) to the file job store +[2020-11-27T14:13:27-0800] [MainThread] [I] [toil.leader] Job ended: 'get_pipeline_inputs' kind-get_pipeline_inputs/instance-q2e4hax4 +[2020-11-27T14:13:27-0800] [MainThread] [I] [toil.leader] Issued job 'launch_protect' kind-launch_protect/instance-oyefziao with job batch system ID: 30 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:13:27-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:13:27: Obtaining file (star:index) to the file job store +[2020-11-27T14:13:28-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpfmwyrj4p/worker_log.txt +[2020-11-27T14:13:29-0800] [MainThread] [I] [toil.leader] Job ended: 'launch_protect' kind-launch_protect/instance-oyefziao +[2020-11-27T14:13:29-0800] [MainThread] [I] [toil.leader] Issued job 'prepare_samples' kind-prepare_samples/instance-qyxpumun with job batch system ID: 31 and cores: 1, disk: 40.0 G, and memory: 2.0 G +[2020-11-27T14:13:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpy7ce3o6i/worker_log.txt +[2020-11-27T14:13:37-0800] [MainThread] [I] [toil.leader] Job ended: 'prepare_samples' kind-prepare_samples/instance-qyxpumun +[2020-11-27T14:13:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_patient_fastqs' kind-get_patient_fastqs/instance-0csdd7bl with job batch system ID: 32 and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-11-27T14:13:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_patient_fastqs' kind-get_patient_fastqs/instance-2pjao3y3 with job batch system ID: 33 and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-11-27T14:13:37-0800] [MainThread] [I] [toil.leader] Issued job 'get_patient_fastqs' kind-get_patient_fastqs/instance-ttmsgsd1 with job batch system ID: 34 and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-11-27T14:13:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:13:38: Downloading Inputs for TEST +[2020-11-27T14:13:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:13:38: Obtaining file (TEST:normal_dna_fastq_2) to the file job store +[2020-11-27T14:13:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:13:38: Obtaining file (TEST:tumor_dna_fastq_2) to the file job store +[2020-11-27T14:13:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:13:38: Obtaining file (TEST:tumor_rna_fastq_1) to the file job store +[2020-11-27T14:13:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:13:38: Obtaining file (TEST:tumor_dna_fastq_1) to the file job store +[2020-11-27T14:13:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:13:38: Obtaining file (TEST:normal_dna_fastq_1) to the file job store +[2020-11-27T14:13:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 14:13:38: Obtaining file (TEST:tumor_rna_fastq_2) to the file job store +[2020-11-27T14:13:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpwahoz1qk/worker_log.txt +[2020-11-27T14:13:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmprrfrctx5/worker_log.txt +[2020-11-27T14:13:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpj3z9sfbp/worker_log.txt +[2020-11-27T14:13:39-0800] [MainThread] [I] [toil.leader] Job ended: 'get_patient_fastqs' kind-get_patient_fastqs/instance-ttmsgsd1 +[2020-11-27T14:13:39-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-tm0pcusm with job batch system ID: 35 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T14:13:39-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-o9p20i2e with job batch system ID: 36 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T14:13:39-0800] [MainThread] [I] [toil.leader] Job ended: 'get_patient_fastqs' kind-get_patient_fastqs/instance-0csdd7bl +[2020-11-27T14:13:39-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-czlzbjjr with job batch system ID: 37 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T14:13:39-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-5drwan9s with job batch system ID: 38 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-27T14:13:39-0800] [MainThread] [I] [toil.leader] Job ended: 'get_patient_fastqs' kind-get_patient_fastqs/instance-2pjao3y3 +[2020-11-27T14:13:39-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-p8idulz4 with job batch system ID: 39 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T14:13:39-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-gxt8is74 with job batch system ID: 40 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-27T14:13:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp98lcpcfc/worker_log.txt +[2020-11-27T14:13:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7lzrynh4/worker_log.txt +[2020-11-27T14:13:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp5b4dsnwo/worker_log.txt +[2020-11-27T14:13:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpeort4yjt/worker_log.txt +[2020-11-27T14:13:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpeqo_lq7r/worker_log.txt +[2020-11-27T14:13:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpipu5fbgz/worker_log.txt +[2020-11-27T14:13:40-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-o9p20i2e +[2020-11-27T14:13:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_cutadapt' kind-run_cutadapt/instance-yutysx82 with job batch system ID: 41 and cores: 1, disk: 1.8 G, and memory: 2.0 G +[2020-11-27T14:13:40-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-tm0pcusm +[2020-11-27T14:13:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-m9yl77yk with job batch system ID: 42 and cores: 80, disk: 8.2 G, and memory: 2.0 G +[2020-11-27T14:13:40-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-p8idulz4 +[2020-11-27T14:13:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-d6snzyih with job batch system ID: 43 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-27T14:13:40-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-czlzbjjr +[2020-11-27T14:13:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-v64rszch with job batch system ID: 44 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-27T14:13:40-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-5drwan9s +[2020-11-27T14:13:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-fcmrk5t5 with job batch system ID: 45 and cores: 80, disk: 13.4 G, and memory: 12.0 G +[2020-11-27T14:13:40-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-gxt8is74 +[2020-11-27T14:13:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-7e3ubnz3 with job batch system ID: 46 and cores: 80, disk: 12.8 G, and memory: 12.0 G +[2020-11-27T14:13:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp0l35ublp/worker_log.txt +[2020-11-27T14:13:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpl7ma4hvi/worker_log.txt +[2020-11-27T14:13:41-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_cutadapt file:/scratch/drkthomp/jobStore kind-run_cutadapt/instance-yutysx82. +[2020-11-27T14:13:41-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_cutadapt' kind-run_cutadapt/instance-yutysx82 +[2020-11-27T14:13:41-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_cutadapt' kind-run_cutadapt/instance-yutysx82 +[2020-11-27T14:13:41-0800] [MainThread] [W] [toil.leader] Log from job kind-run_cutadapt/instance-yutysx82 follows: +=========> + [2020-11-27T14:13:41-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T14:13:41-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/qc/rna.py", line 59, in run_cutadapt + dockerhub=univ_options['dockerhub'], tool_version=cutadapt_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-27T14:13:41-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-27T14:13:41-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_cutadapt' kind-run_cutadapt/instance-yutysx82 with ID kind-run_cutadapt/instance-yutysx82 to 1 +[2020-11-27T14:13:41-0800] [MainThread] [W] [toil.jobGraph] We have increased the disk of the failed job 'run_cutadapt' kind-run_cutadapt/instance-yutysx82 to the default of 2147483648 bytes +[2020-11-27T14:13:41-0800] [MainThread] [I] [toil.leader] Issued job 'run_cutadapt' kind-run_cutadapt/instance-yutysx82 with job batch system ID: 47 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T14:13:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp824t788m/worker_log.txt +[2020-11-27T14:15:00-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-v64rszch. +[2020-11-27T14:15:00-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-v64rszch +[2020-11-27T14:15:00-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-v64rszch +[2020-11-27T14:15:00-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-v64rszch follows: +=========> + [2020-11-27T14:13:42-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T14:13:42-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-27T14:14:52-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp824t788m/c94725e7-74b1-4ad5-beb6-9d2843ce6234/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp824t788m/c94725e7-74b1-4ad5-beb6-9d2843ce6234/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp824t788m/c94725e7-74b1-4ad5-beb6-9d2843ce6234/phlat_index.tar.gz'} +<========= +[2020-11-27T14:15:00-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-v64rszch with ID kind-run_phlat/instance-v64rszch to 1 +[2020-11-27T14:15:00-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-v64rszch with job batch system ID: 48 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-27T14:15:00-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-m9yl77yk. +[2020-11-27T14:15:00-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-m9yl77yk +[2020-11-27T14:15:00-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-m9yl77yk +[2020-11-27T14:15:00-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-m9yl77yk follows: +=========> + [2020-11-27T14:13:41-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T14:13:41-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-27T14:14:51-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp0l35ublp/71262221-6a92-49c6-aeb8-49eb3b8a0a66/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp0l35ublp/71262221-6a92-49c6-aeb8-49eb3b8a0a66/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp0l35ublp/71262221-6a92-49c6-aeb8-49eb3b8a0a66/phlat_index.tar.gz'} +<========= +[2020-11-27T14:15:00-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-m9yl77yk with ID kind-run_phlat/instance-m9yl77yk to 1 +[2020-11-27T14:15:00-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-m9yl77yk with job batch system ID: 49 and cores: 80, disk: 8.2 G, and memory: 2.0 G +[2020-11-27T14:15:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpvo1wymnh/worker_log.txt +[2020-11-27T14:15:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp69kcvz3e/worker_log.txt +[2020-11-27T14:16:11-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-d6snzyih. +[2020-11-27T14:16:11-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-d6snzyih +[2020-11-27T14:16:11-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-d6snzyih +[2020-11-27T14:16:11-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-d6snzyih follows: +=========> + [2020-11-27T14:15:01-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T14:15:01-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-27T14:16:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpvo1wymnh/f02da2f7-431d-4032-b57b-4d7a25ca5a2f/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpvo1wymnh/f02da2f7-431d-4032-b57b-4d7a25ca5a2f/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpvo1wymnh/f02da2f7-431d-4032-b57b-4d7a25ca5a2f/phlat_index.tar.gz'} +<========= +[2020-11-27T14:16:11-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-d6snzyih with ID kind-run_phlat/instance-d6snzyih to 1 +[2020-11-27T14:16:11-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-d6snzyih with job batch system ID: 50 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-27T14:16:12-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1fawerqj/worker_log.txt +[2020-11-27T14:16:12-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_cutadapt file:/scratch/drkthomp/jobStore kind-run_cutadapt/instance-yutysx82. +[2020-11-27T14:16:12-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_cutadapt' kind-run_cutadapt/instance-yutysx82 +[2020-11-27T14:16:12-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_cutadapt' kind-run_cutadapt/instance-yutysx82 +[2020-11-27T14:16:12-0800] [MainThread] [W] [toil.leader] Log from job kind-run_cutadapt/instance-yutysx82 follows: +=========> + [2020-11-27T14:16:12-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T14:16:12-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/qc/rna.py", line 59, in run_cutadapt + dockerhub=univ_options['dockerhub'], tool_version=cutadapt_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-27T14:16:12-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-27T14:16:12-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_cutadapt' kind-run_cutadapt/instance-yutysx82 with ID kind-run_cutadapt/instance-yutysx82 to 0 +[2020-11-27T14:16:12-0800] [MainThread] [W] [toil.leader] Job 'run_cutadapt' kind-run_cutadapt/instance-yutysx82 with ID kind-run_cutadapt/instance-yutysx82 is completely failed +[2020-11-27T14:16:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgcvuxymi/worker_log.txt +[2020-11-27T14:16:46-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_bwa file:/scratch/drkthomp/jobStore kind-run_bwa/instance-fcmrk5t5. +[2020-11-27T14:16:46-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_bwa' kind-run_bwa/instance-fcmrk5t5 +[2020-11-27T14:16:46-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_bwa' kind-run_bwa/instance-fcmrk5t5 +[2020-11-27T14:16:46-0800] [MainThread] [W] [toil.leader] Log from job kind-run_bwa/instance-fcmrk5t5 follows: +=========> + [2020-11-27T14:15:01-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T14:15:01-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/alignment/dna.py", line 142, in run_bwa + tool_version=bwa_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-27T14:16:45-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-27T14:16:46-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_bwa' kind-run_bwa/instance-fcmrk5t5 with ID kind-run_bwa/instance-fcmrk5t5 to 1 +[2020-11-27T14:16:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-fcmrk5t5 with job batch system ID: 51 and cores: 80, disk: 13.4 G, and memory: 12.0 G +[2020-11-27T14:16:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmplh6lw_me/worker_log.txt +[2020-11-27T14:17:24-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-d6snzyih. +[2020-11-27T14:17:24-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-d6snzyih +[2020-11-27T14:17:24-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-d6snzyih +[2020-11-27T14:17:24-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-d6snzyih follows: +=========> + [2020-11-27T14:16:13-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T14:16:13-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-27T14:17:22-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgcvuxymi/c7401915-c7ef-42bf-8a49-e6dc12723fed/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgcvuxymi/c7401915-c7ef-42bf-8a49-e6dc12723fed/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgcvuxymi/c7401915-c7ef-42bf-8a49-e6dc12723fed/phlat_index.tar.gz'} +<========= +[2020-11-27T14:17:24-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-d6snzyih with ID kind-run_phlat/instance-d6snzyih to 0 +[2020-11-27T14:17:24-0800] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-d6snzyih with ID kind-run_phlat/instance-d6snzyih is completely failed +[2020-11-27T14:17:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdascl6ga/worker_log.txt +[2020-11-27T14:17:58-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-v64rszch. +[2020-11-27T14:17:58-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-v64rszch +[2020-11-27T14:17:58-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-v64rszch +[2020-11-27T14:17:58-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-v64rszch follows: +=========> + [2020-11-27T14:16:47-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T14:16:47-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-27T14:17:57-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmplh6lw_me/41b69a31-ef48-4fd5-88bb-59fd0607a159/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmplh6lw_me/41b69a31-ef48-4fd5-88bb-59fd0607a159/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmplh6lw_me/41b69a31-ef48-4fd5-88bb-59fd0607a159/phlat_index.tar.gz'} +<========= +[2020-11-27T14:17:58-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-v64rszch with ID kind-run_phlat/instance-v64rszch to 0 +[2020-11-27T14:17:58-0800] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-v64rszch with ID kind-run_phlat/instance-v64rszch is completely failed +[2020-11-27T14:17:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpjitzglb4/worker_log.txt +[2020-11-27T14:18:34-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_phlat file:/scratch/drkthomp/jobStore kind-run_phlat/instance-m9yl77yk. +[2020-11-27T14:18:34-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-m9yl77yk +[2020-11-27T14:18:34-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-run_phlat/instance-m9yl77yk +[2020-11-27T14:18:34-0800] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-m9yl77yk follows: +=========> + [2020-11-27T14:17:24-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T14:17:24-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-27T14:18:33-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdascl6ga/e193b8c5-df90-4f55-af24-6aecef7d9917/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdascl6ga/e193b8c5-df90-4f55-af24-6aecef7d9917/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdascl6ga/e193b8c5-df90-4f55-af24-6aecef7d9917/phlat_index.tar.gz'} +<========= +[2020-11-27T14:18:34-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' kind-run_phlat/instance-m9yl77yk with ID kind-run_phlat/instance-m9yl77yk to 0 +[2020-11-27T14:18:34-0800] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-m9yl77yk with ID kind-run_phlat/instance-m9yl77yk is completely failed +[2020-11-27T14:18:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpfwls_luy/worker_log.txt +[2020-11-27T14:19:41-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_bwa file:/scratch/drkthomp/jobStore kind-run_bwa/instance-fcmrk5t5. +[2020-11-27T14:19:41-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_bwa' kind-run_bwa/instance-fcmrk5t5 +[2020-11-27T14:19:41-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_bwa' kind-run_bwa/instance-fcmrk5t5 +[2020-11-27T14:19:41-0800] [MainThread] [W] [toil.leader] Log from job kind-run_bwa/instance-fcmrk5t5 follows: +=========> + [2020-11-27T14:17:59-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T14:17:59-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/alignment/dna.py", line 142, in run_bwa + tool_version=bwa_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-27T14:19:39-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-27T14:19:41-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_bwa' kind-run_bwa/instance-fcmrk5t5 with ID kind-run_bwa/instance-fcmrk5t5 to 0 +[2020-11-27T14:19:41-0800] [MainThread] [W] [toil.leader] Job 'run_bwa' kind-run_bwa/instance-fcmrk5t5 with ID kind-run_bwa/instance-fcmrk5t5 is completely failed +[2020-11-27T14:20:12-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_bwa file:/scratch/drkthomp/jobStore kind-run_bwa/instance-7e3ubnz3. +[2020-11-27T14:20:12-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_bwa' kind-run_bwa/instance-7e3ubnz3 +[2020-11-27T14:20:12-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_bwa' kind-run_bwa/instance-7e3ubnz3 +[2020-11-27T14:20:12-0800] [MainThread] [W] [toil.leader] Log from job kind-run_bwa/instance-7e3ubnz3 follows: +=========> + [2020-11-27T14:18:35-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T14:18:35-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/alignment/dna.py", line 142, in run_bwa + tool_version=bwa_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-27T14:20:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-27T14:20:12-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_bwa' kind-run_bwa/instance-7e3ubnz3 with ID kind-run_bwa/instance-7e3ubnz3 to 1 +[2020-11-27T14:20:12-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-7e3ubnz3 with job batch system ID: 52 and cores: 80, disk: 12.8 G, and memory: 12.0 G +[2020-11-27T14:20:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpz_y36x7c/worker_log.txt +[2020-11-27T14:21:54-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_bwa file:/scratch/drkthomp/jobStore kind-run_bwa/instance-7e3ubnz3. +[2020-11-27T14:21:54-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_bwa' kind-run_bwa/instance-7e3ubnz3 +[2020-11-27T14:21:54-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_bwa' kind-run_bwa/instance-7e3ubnz3 +[2020-11-27T14:21:54-0800] [MainThread] [W] [toil.leader] Log from job kind-run_bwa/instance-7e3ubnz3 follows: +=========> + [2020-11-27T14:20:13-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T14:20:13-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/alignment/dna.py", line 142, in run_bwa + tool_version=bwa_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-27T14:21:52-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-27T14:21:54-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_bwa' kind-run_bwa/instance-7e3ubnz3 with ID kind-run_bwa/instance-7e3ubnz3 to 0 +[2020-11-27T14:21:54-0800] [MainThread] [W] [toil.leader] Job 'run_bwa' kind-run_bwa/instance-7e3ubnz3 with ID kind-run_bwa/instance-7e3ubnz3 is completely failed +[2020-11-27T14:22:07-0800] [MainThread] [I] [toil.leader] Finished toil run with 18 failed jobs. +[2020-11-27T14:22:07-0800] [MainThread] [I] [toil.leader] Failed jobs at end of the run: 'run_phlat' kind-EncapsulatedJob/instance-tm0pcusm 'prepare_samples' kind-prepare_samples/instance-qyxpumun 'run_cutadapt' kind-EncapsulatedJob/instance-o9p20i2e 'get_patient_fastqs' kind-get_patient_fastqs/instance-0csdd7bl 'run_bwa' kind-run_bwa/instance-fcmrk5t5 'run_bwa' kind-EncapsulatedJob/instance-gxt8is74 'run_bwa' kind-EncapsulatedJob/instance-5drwan9s 'get_patient_fastqs' kind-get_patient_fastqs/instance-2pjao3y3 'run_cutadapt' kind-run_cutadapt/instance-yutysx82 'get_patient_fastqs' kind-get_patient_fastqs/instance-ttmsgsd1 'run_phlat' kind-EncapsulatedJob/instance-czlzbjjr 'run_phlat' kind-run_phlat/instance-v64rszch 'get_all_tool_inputs' kind-parse_config_file/instance-utcdr1zk 'run_phlat' kind-run_phlat/instance-d6snzyih 'launch_protect' kind-launch_protect/instance-oyefziao 'run_phlat' kind-run_phlat/instance-m9yl77yk 'run_bwa' kind-run_bwa/instance-7e3ubnz3 'run_phlat' kind-EncapsulatedJob/instance-p8idulz4 +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/bin/ProTECT", line 11, in + load_entry_point('protect', 'console_scripts', 'ProTECT')() + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1028, in main + Job.Runner.startToil(start, params) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 796, in startToil + return toil.start(job) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 846, in start + return self._runMainLoop(rootJobGraph) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 1138, in _runMainLoop + jobCache=self._jobCache).run() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/leader.py", line 269, in run + raise FailedJobsException(self.config.jobStore, self.toilState.totalFailedJobs, self.jobStore) +toil.leader.FailedJobsException: The job store 'file:/scratch/drkthomp/jobStore' contains 18 failed jobs: 'run_phlat' kind-EncapsulatedJob/instance-tm0pcusm, 'prepare_samples' kind-prepare_samples/instance-qyxpumun, 'run_cutadapt' kind-EncapsulatedJob/instance-o9p20i2e, 'get_patient_fastqs' kind-get_patient_fastqs/instance-0csdd7bl, 'run_bwa' kind-run_bwa/instance-fcmrk5t5, 'run_bwa' kind-EncapsulatedJob/instance-gxt8is74, 'run_bwa' kind-EncapsulatedJob/instance-5drwan9s, 'get_patient_fastqs' kind-get_patient_fastqs/instance-2pjao3y3, 'run_cutadapt' kind-run_cutadapt/instance-yutysx82, 'get_patient_fastqs' kind-get_patient_fastqs/instance-ttmsgsd1, 'run_phlat' kind-EncapsulatedJob/instance-czlzbjjr, 'run_phlat' kind-run_phlat/instance-v64rszch, 'get_all_tool_inputs' kind-parse_config_file/instance-utcdr1zk, 'run_phlat' kind-run_phlat/instance-d6snzyih, 'launch_protect' kind-launch_protect/instance-oyefziao, 'run_phlat' kind-run_phlat/instance-m9yl77yk, 'run_bwa' kind-run_bwa/instance-7e3ubnz3, 'run_phlat' kind-EncapsulatedJob/instance-p8idulz4 +Log from job 'run_bwa' kind-run_bwa/instance-fcmrk5t5 follows: +=========> + [2020-11-27T14:17:59-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T14:17:59-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/alignment/dna.py", line 142, in run_bwa + tool_version=bwa_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-27T14:19:39-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +Log from job 'run_cutadapt' kind-run_cutadapt/instance-yutysx82 follows: +=========> + [2020-11-27T14:16:12-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T14:16:12-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/qc/rna.py", line 59, in run_cutadapt + dockerhub=univ_options['dockerhub'], tool_version=cutadapt_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-27T14:16:12-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +Log from job 'run_phlat' kind-run_phlat/instance-v64rszch follows: +=========> + [2020-11-27T14:16:47-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T14:16:47-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-27T14:17:57-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmplh6lw_me/41b69a31-ef48-4fd5-88bb-59fd0607a159/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmplh6lw_me/41b69a31-ef48-4fd5-88bb-59fd0607a159/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmplh6lw_me/41b69a31-ef48-4fd5-88bb-59fd0607a159/phlat_index.tar.gz'} +<========= +Log from job 'run_phlat' kind-run_phlat/instance-d6snzyih follows: +=========> + [2020-11-27T14:16:13-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T14:16:13-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-27T14:17:22-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgcvuxymi/c7401915-c7ef-42bf-8a49-e6dc12723fed/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgcvuxymi/c7401915-c7ef-42bf-8a49-e6dc12723fed/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgcvuxymi/c7401915-c7ef-42bf-8a49-e6dc12723fed/phlat_index.tar.gz'} +<========= +Log from job 'run_phlat' kind-run_phlat/instance-m9yl77yk follows: +=========> + [2020-11-27T14:17:24-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T14:17:24-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 74, in run_phlat + dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-27T14:18:33-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + {'input_1.fastq': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdascl6ga/e193b8c5-df90-4f55-af24-6aecef7d9917/input_1.fastq', 'input_2.fastq': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdascl6ga/e193b8c5-df90-4f55-af24-6aecef7d9917/input_2.fastq', 'phlat_index.tar.gz': '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdascl6ga/e193b8c5-df90-4f55-af24-6aecef7d9917/phlat_index.tar.gz'} +<========= +Log from job 'run_bwa' kind-run_bwa/instance-7e3ubnz3 follows: +=========> + [2020-11-27T14:20:13-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T14:20:13-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get http://%2Fvar%2Frun%2Fdocker.sock/v1.37/images/json: dial unix /var/run/docker.sock: connect: permission denied + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/alignment/dna.py", line 142, in run_bwa + tool_version=bwa_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 113, in docker_call + dimg_rv = subprocess.check_output(call).decode('utf-8') + File "/usr/lib64/python3.6/subprocess.py", line 336, in check_output + **kwargs).stdout + File "/usr/lib64/python3.6/subprocess.py", line 418, in run + output=stdout, stderr=stderr) + subprocess.CalledProcessError: Command '['docker', 'images']' returned non-zero exit status 1. + [2020-11-27T14:21:52-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= diff --git a/errors/2020-11-27-15-11-20.txt b/errors/2020-11-27-15-11-20.txt new file mode 100644 index 00000000..254887a7 --- /dev/null +++ b/errors/2020-11-27-15-11-20.txt @@ -0,0 +1,19 @@ +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 104, in initialize + os.mkdir(self.jobStoreDir) +FileExistsError: [Errno 17] File exists: '/scratch/drkthomp/jobStore' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/bin/ProTECT", line 11, in + load_entry_point('protect', 'console_scripts', 'ProTECT')() + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1028, in main + Job.Runner.startToil(start, params) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 794, in startToil + with Toil(options) as toil: + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 774, in __enter__ + jobStore.initialize(config) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/jobStores/fileJobStore.py", line 107, in initialize + raise JobStoreExistsException(self.jobStoreDir) +toil.jobStores.abstractJobStore.JobStoreExistsException: The job store '/scratch/drkthomp/jobStore' already exists. Use --restart to resume the workflow, or remove the job store with 'toil clean' to start the workflow from scratch. diff --git a/errors/2020-11-27-15-11-35.txt b/errors/2020-11-27-15-11-35.txt new file mode 100644 index 00000000..05182c99 --- /dev/null +++ b/errors/2020-11-27-15-11-35.txt @@ -0,0 +1,1813 @@ +[2020-11-27T15:11:36-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_cutadapt/instance-yutysx82 +[2020-11-27T15:11:36-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-parse_config_file/instance-utcdr1zk +[2020-11-27T15:11:36-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_bwa/instance-7e3ubnz3 +[2020-11-27T15:11:36-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-v64rszch +[2020-11-27T15:11:36-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-d6snzyih +[2020-11-27T15:11:36-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-m9yl77yk +[2020-11-27T15:11:36-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_bwa/instance-fcmrk5t5 +[2020-11-27T15:11:36-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-27T15:11:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-v64rszch with job batch system ID: 0 and cores: 80, disk: 8.4 G, and memory: 2.0 G +[2020-11-27T15:11:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-fcmrk5t5 with job batch system ID: 1 and cores: 80, disk: 13.4 G, and memory: 12.0 G +[2020-11-27T15:11:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-d6snzyih with job batch system ID: 2 and cores: 80, disk: 8.3 G, and memory: 2.0 G +[2020-11-27T15:11:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-7e3ubnz3 with job batch system ID: 3 and cores: 80, disk: 12.8 G, and memory: 12.0 G +[2020-11-27T15:11:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-m9yl77yk with job batch system ID: 4 and cores: 80, disk: 8.2 G, and memory: 2.0 G +[2020-11-27T15:11:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_cutadapt' kind-run_cutadapt/instance-yutysx82 with job batch system ID: 5 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T15:11:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmphoahrdix/worker_log.txt +[2020-11-27T15:11:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp9x9idqob/worker_log.txt +[2020-11-27T15:11:38-0800] [MainThread] [I] [toil.leader] 2 jobs are running, 4 jobs are issued and waiting to run +[2020-11-27T15:14:30-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:14:30: Ran bwa on TEST:tumor_dna successfully +[2020-11-27T15:14:31-0800] [MainThread] [I] [toil.leader] Job ended: 'run_bwa' kind-run_bwa/instance-fcmrk5t5 +[2020-11-27T15:14:32-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-lvaegcmc with job batch system ID: 6 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:14:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpav3w7ox8/worker_log.txt +[2020-11-27T15:17:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:17:17: Ran bwa on TEST:normal_dna successfully +[2020-11-27T15:17:19-0800] [MainThread] [I] [toil.leader] Job ended: 'run_bwa' kind-run_bwa/instance-7e3ubnz3 +[2020-11-27T15:17:19-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ykuu0crz with job batch system ID: 7 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:17:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpoamyqz4b/worker_log.txt +[2020-11-27T15:17:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpueqoh16x/worker_log.txt +[2020-11-27T15:17:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpd1t0cjkm/worker_log.txt +[2020-11-27T15:17:20-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-lvaegcmc +[2020-11-27T15:17:20-0800] [MainThread] [I] [toil.leader] Issued job 'bam_conversion' kind-bam_conversion/instance-j6ma_por with job batch system ID: 8 and cores: 1, disk: 1.7 G, and memory: 12.0 G +[2020-11-27T15:17:20-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ykuu0crz +[2020-11-27T15:17:20-0800] [MainThread] [I] [toil.leader] Issued job 'bam_conversion' kind-bam_conversion/instance-jkzbr7mz with job batch system ID: 9 and cores: 1, disk: 1.6 G, and memory: 12.0 G +[2020-11-27T15:17:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1k0ctpgp/worker_log.txt +[2020-11-27T15:17:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpkncz_kp1/worker_log.txt +[2020-11-27T15:18:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:18:16: Ran cutadapt on TEST successfully +[2020-11-27T15:18:16-0800] [MainThread] [I] [toil.leader] Job ended: 'run_cutadapt' kind-run_cutadapt/instance-yutysx82 +[2020-11-27T15:18:16-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-wizon1te with job batch system ID: 10 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:18:17-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpijmjml1x/worker_log.txt +[2020-11-27T15:18:17-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-wizon1te +[2020-11-27T15:18:17-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-y_lo_bvx with job batch system ID: 11 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-27T15:18:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpcsemuufl/worker_log.txt +[2020-11-27T15:18:18-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-y_lo_bvx +[2020-11-27T15:18:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_star' kind-run_star/instance-qh34obz5 with job batch system ID: 12 and cores: 80, disk: 53.7 G, and memory: 48.6 G +[2020-11-27T15:18:32-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:18:32: Ran sam2bam on TEST:normal_dna successfully +[2020-11-27T15:18:32-0800] [MainThread] [I] [toil.leader] Job ended: 'bam_conversion' kind-bam_conversion/instance-jkzbr7mz +[2020-11-27T15:18:32-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-wcf4pqqf with job batch system ID: 13 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:18:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmphoufwt6j/worker_log.txt +[2020-11-27T15:18:33-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-wcf4pqqf +[2020-11-27T15:18:33-0800] [MainThread] [I] [toil.leader] Issued job 'fix_bam_header' kind-fix_bam_header/instance-ienughz0 with job batch system ID: 14 and cores: 1, disk: 1.6 G, and memory: 12.0 G +[2020-11-27T15:18:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpofb5ep8r/worker_log.txt +[2020-11-27T15:18:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:18:39: Ran reheader on TEST:normal_dna successfully +[2020-11-27T15:18:39-0800] [MainThread] [I] [toil.leader] Job ended: 'fix_bam_header' kind-fix_bam_header/instance-ienughz0 +[2020-11-27T15:18:39-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ghcbwsnm with job batch system ID: 15 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:18:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmprnf5n3dc/worker_log.txt +[2020-11-27T15:18:40-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ghcbwsnm +[2020-11-27T15:18:40-0800] [MainThread] [I] [toil.leader] Issued job 'add_readgroups' kind-add_readgroups/instance-p6bwwjtc with job batch system ID: 16 and cores: 1, disk: 1.3 G, and memory: 12.0 G +[2020-11-27T15:18:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmppl1qrp3j/worker_log.txt +[2020-11-27T15:18:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:18:46: Ran sam2bam on TEST:tumor_dna successfully +[2020-11-27T15:18:46-0800] [MainThread] [I] [toil.leader] Job ended: 'bam_conversion' kind-bam_conversion/instance-j6ma_por +[2020-11-27T15:18:46-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-l7xcwyxl with job batch system ID: 17 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:18:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpak1fo0uj/worker_log.txt +[2020-11-27T15:18:47-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-l7xcwyxl +[2020-11-27T15:18:47-0800] [MainThread] [I] [toil.leader] Issued job 'fix_bam_header' kind-fix_bam_header/instance-wt52lipa with job batch system ID: 18 and cores: 1, disk: 1.7 G, and memory: 12.0 G +[2020-11-27T15:18:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmptvibe8w8/worker_log.txt +[2020-11-27T15:18:53-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:18:53: Ran reheader on TEST:tumor_dna successfully +[2020-11-27T15:18:53-0800] [MainThread] [I] [toil.leader] Job ended: 'fix_bam_header' kind-fix_bam_header/instance-wt52lipa +[2020-11-27T15:18:53-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ud3eaclr with job batch system ID: 19 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:18:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpmeudk4cr/worker_log.txt +[2020-11-27T15:18:54-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ud3eaclr +[2020-11-27T15:18:54-0800] [MainThread] [I] [toil.leader] Issued job 'add_readgroups' kind-add_readgroups/instance-2q908lp_ with job batch system ID: 20 and cores: 1, disk: 1.5 G, and memory: 12.0 G +[2020-11-27T15:18:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpyhayq_5p/worker_log.txt +[2020-11-27T15:19:08-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:19:08: Ran phlat on TEST:tumor_dna successfully +[2020-11-27T15:19:09-0800] [MainThread] [I] [toil.leader] Job ended: 'run_phlat' kind-run_phlat/instance-v64rszch +[2020-11-27T15:19:09-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-7s1jyb6z with job batch system ID: 21 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:19:10-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpeze8s6w2/worker_log.txt +[2020-11-27T15:19:10-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpq9ie885j/worker_log.txt +[2020-11-27T15:19:10-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-7s1jyb6z +[2020-11-27T15:20:13-0800] [MainThread] [I] [toil.leader] Job ended: 'add_readgroups' kind-add_readgroups/instance-p6bwwjtc +[2020-11-27T15:20:13-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-pe512dl4 with job batch system ID: 22 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:20:13-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:20:13: Ran add_read_groups on TEST:normal_dna successfully +[2020-11-27T15:20:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpaunpq5ol/worker_log.txt +[2020-11-27T15:20:14-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-pe512dl4 +[2020-11-27T15:20:14-0800] [MainThread] [I] [toil.leader] Issued job 'mark_duplicates' kind-mark_duplicates/instance-bbn2ydoj with job batch system ID: 23 and cores: 1, disk: 1.3 G, and memory: 12.0 G +[2020-11-27T15:20:15-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp5foc_nk1/worker_log.txt +[2020-11-27T15:20:52-0800] [MainThread] [I] [toil.leader] Job ended: 'add_readgroups' kind-add_readgroups/instance-2q908lp_ +[2020-11-27T15:20:52-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-gr7c4s50 with job batch system ID: 24 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:20:52-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:20:52: Ran add_read_groups on TEST:tumor_dna successfully +[2020-11-27T15:20:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpsskpugxz/worker_log.txt +[2020-11-27T15:20:53-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-gr7c4s50 +[2020-11-27T15:20:53-0800] [MainThread] [I] [toil.leader] Issued job 'mark_duplicates' kind-mark_duplicates/instance-z2euuvkq with job batch system ID: 25 and cores: 1, disk: 1.5 G, and memory: 12.0 G +[2020-11-27T15:20:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpmcsp03au/worker_log.txt +[2020-11-27T15:22:42-0800] [MainThread] [I] [toil.leader] Job ended: 'mark_duplicates' kind-mark_duplicates/instance-bbn2ydoj +[2020-11-27T15:22:42-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-g5kkkb8e with job batch system ID: 26 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:22:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:22:42: Ran mark_duplicates on TEST:normal_dna successfully +[2020-11-27T15:22:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp6j5f403t/worker_log.txt +[2020-11-27T15:22:43-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-g5kkkb8e +[2020-11-27T15:22:43-0800] [MainThread] [I] [toil.leader] Issued job 'index_bamfile' kind-index_bamfile/instance-8u984bpp with job batch system ID: 27 and cores: 1, disk: 360.6 M, and memory: 12.0 G +[2020-11-27T15:22:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp6i6fnhqg/worker_log.txt +[2020-11-27T15:23:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:23:07: Exporting files/for-job/kind-mark_duplicates/instance-bbn2ydoj/file-h5w6b6jg/normal_dna_mkdup.bam to output location +[2020-11-27T15:23:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:23:07: Exporting files/for-job/kind-index_bamfile/instance-8u984bpp/file-rx5r3_fe/normal_dna_fix_pg_sorted.bam.bai to output location +[2020-11-27T15:23:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:23:07: Ran samtools-index on TEST:normal_dna successfully +[2020-11-27T15:23:07-0800] [MainThread] [I] [toil.leader] Job ended: 'index_bamfile' kind-index_bamfile/instance-8u984bpp +[2020-11-27T15:23:07-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-42dq9b3d with job batch system ID: 28 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:23:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpvygj0pl1/worker_log.txt +[2020-11-27T15:23:08-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-42dq9b3d +[2020-11-27T15:23:08-0800] [MainThread] [I] [toil.leader] Issued job 'index_bamfile' kind-Job/instance-g5kkkb8e with job batch system ID: 29 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:23:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmprsvcsw72/worker_log.txt +[2020-11-27T15:23:09-0800] [MainThread] [I] [toil.leader] Job ended: 'index_bamfile' kind-Job/instance-g5kkkb8e +[2020-11-27T15:23:09-0800] [MainThread] [I] [toil.leader] Issued job 'mark_duplicates' kind-Job/instance-pe512dl4 with job batch system ID: 30 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:23:10-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpjm5sqqzv/worker_log.txt +[2020-11-27T15:23:10-0800] [MainThread] [I] [toil.leader] Job ended: 'mark_duplicates' kind-Job/instance-pe512dl4 +[2020-11-27T15:23:10-0800] [MainThread] [I] [toil.leader] Issued job 'add_readgroups' kind-Job/instance-ghcbwsnm with job batch system ID: 31 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:23:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpsbkt61k4/worker_log.txt +[2020-11-27T15:23:11-0800] [MainThread] [I] [toil.leader] Job ended: 'add_readgroups' kind-Job/instance-ghcbwsnm +[2020-11-27T15:23:11-0800] [MainThread] [I] [toil.leader] Issued job 'fix_bam_header' kind-Job/instance-wcf4pqqf with job batch system ID: 32 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:23:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpxm8c9oox/worker_log.txt +[2020-11-27T15:23:11-0800] [MainThread] [I] [toil.leader] Job ended: 'fix_bam_header' kind-Job/instance-wcf4pqqf +[2020-11-27T15:23:11-0800] [MainThread] [I] [toil.leader] Issued job 'bam_conversion' kind-Job/instance-ykuu0crz with job batch system ID: 33 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:23:12-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmplb0rddrg/worker_log.txt +[2020-11-27T15:23:12-0800] [MainThread] [I] [toil.leader] Job ended: 'bam_conversion' kind-Job/instance-ykuu0crz +[2020-11-27T15:23:12-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-hlsgfes5 with job batch system ID: 34 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:23:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpt8yq5pse/worker_log.txt +[2020-11-27T15:23:13-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-hlsgfes5 +[2020-11-27T15:23:37-0800] [MainThread] [I] [toil.leader] Job ended: 'mark_duplicates' kind-mark_duplicates/instance-z2euuvkq +[2020-11-27T15:23:37-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-kkp3gre3 with job batch system ID: 35 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:23:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:23:37: Ran mark_duplicates on TEST:tumor_dna successfully +[2020-11-27T15:23:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpaxst07kf/worker_log.txt +[2020-11-27T15:26:12-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:26:12: Ran phlat on TEST:normal_dna successfully +[2020-11-27T15:26:12-0800] [MainThread] [I] [toil.leader] Job ended: 'run_phlat' kind-run_phlat/instance-d6snzyih +[2020-11-27T15:26:12-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-5x2sz54b with job batch system ID: 36 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:26:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpnjjfy7ol/worker_log.txt +[2020-11-27T15:29:47-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:29:47: Ran phlat on TEST:tumor_rna successfully +[2020-11-27T15:29:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_phlat' kind-run_phlat/instance-m9yl77yk +[2020-11-27T15:29:48-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-p8u5n56k with job batch system ID: 37 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:29:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1r2nvzom/worker_log.txt +[2020-11-27T15:29:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpprvtw4se/worker_log.txt +[2020-11-27T15:29:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpu0_0obtq/worker_log.txt +[2020-11-27T15:29:49-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-5x2sz54b +[2020-11-27T15:29:49-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-p8u5n56k +[2020-11-27T15:29:49-0800] [MainThread] [I] [toil.leader] Issued job 'merge_phlat_calls' kind-merge_phlat_calls/instance-6j0cplsq with job batch system ID: 38 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T15:29:49-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-kkp3gre3 +[2020-11-27T15:29:49-0800] [MainThread] [I] [toil.leader] Issued job 'index_bamfile' kind-index_bamfile/instance-co4_w9p0 with job batch system ID: 39 and cores: 1, disk: 412.9 M, and memory: 12.0 G +[2020-11-27T15:29:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpo0mu1bkj/worker_log.txt +[2020-11-27T15:29:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpcb0tri7r/worker_log.txt +[2020-11-27T15:29:50-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_phlat_calls' kind-merge_phlat_calls/instance-6j0cplsq +[2020-11-27T15:29:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:29:50: Merging Phlat calls +[2020-11-27T15:29:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:29:50: Exporting files/for-job/kind-merge_phlat_calls/instance-6j0cplsq/file-3_g4cltx/mhci_alleles.list to output location +[2020-11-27T15:29:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:29:50: Exporting files/for-job/kind-merge_phlat_calls/instance-6j0cplsq/file-8j1ormjp/mhcii_alleles.list to output location +[2020-11-27T15:30:20-0800] [MainThread] [I] [toil.leader] Job ended: 'index_bamfile' kind-index_bamfile/instance-co4_w9p0 +[2020-11-27T15:30:20-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-z5y81jpv with job batch system ID: 40 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:30:21-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:30:21: Exporting files/for-job/kind-mark_duplicates/instance-z2euuvkq/file-5kuacsv6/tumor_dna_mkdup.bam to output location +[2020-11-27T15:30:21-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:30:21: Exporting files/for-job/kind-index_bamfile/instance-co4_w9p0/file-th5_vxy6/tumor_dna_fix_pg_sorted.bam.bai to output location +[2020-11-27T15:30:21-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:30:21: Ran samtools-index on TEST:tumor_dna successfully +[2020-11-27T15:30:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_oc5sj9s/worker_log.txt +[2020-11-27T15:30:21-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-z5y81jpv +[2020-11-27T15:30:21-0800] [MainThread] [I] [toil.leader] Issued job 'index_bamfile' kind-Job/instance-kkp3gre3 with job batch system ID: 41 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:30:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpmj1gof0z/worker_log.txt +[2020-11-27T15:30:22-0800] [MainThread] [I] [toil.leader] Job ended: 'index_bamfile' kind-Job/instance-kkp3gre3 +[2020-11-27T15:30:22-0800] [MainThread] [I] [toil.leader] Issued job 'mark_duplicates' kind-Job/instance-gr7c4s50 with job batch system ID: 42 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:30:23-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp8b6g7qli/worker_log.txt +[2020-11-27T15:30:23-0800] [MainThread] [I] [toil.leader] Job ended: 'mark_duplicates' kind-Job/instance-gr7c4s50 +[2020-11-27T15:30:23-0800] [MainThread] [I] [toil.leader] Issued job 'add_readgroups' kind-Job/instance-ud3eaclr with job batch system ID: 43 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:30:23-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpyv09413c/worker_log.txt +[2020-11-27T15:30:23-0800] [MainThread] [I] [toil.leader] Job ended: 'add_readgroups' kind-Job/instance-ud3eaclr +[2020-11-27T15:30:23-0800] [MainThread] [I] [toil.leader] Issued job 'fix_bam_header' kind-Job/instance-l7xcwyxl with job batch system ID: 44 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:30:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpunx0imea/worker_log.txt +[2020-11-27T15:30:24-0800] [MainThread] [I] [toil.leader] Job ended: 'fix_bam_header' kind-Job/instance-l7xcwyxl +[2020-11-27T15:30:24-0800] [MainThread] [I] [toil.leader] Issued job 'bam_conversion' kind-Job/instance-lvaegcmc with job batch system ID: 45 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:30:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpc49rorah/worker_log.txt +[2020-11-27T15:30:25-0800] [MainThread] [I] [toil.leader] Job ended: 'bam_conversion' kind-Job/instance-lvaegcmc +[2020-11-27T15:30:25-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-l6jndejt with job batch system ID: 46 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:30:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdylogkp_/worker_log.txt +[2020-11-27T15:30:26-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-l6jndejt +[2020-11-27T15:30:26-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-bj0twc34 with job batch system ID: 47 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-27T15:30:26-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-qi2fqyz_ with job batch system ID: 48 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T15:30:26-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-wlu1vdpy with job batch system ID: 49 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T15:30:26-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-ixd9le0x with job batch system ID: 50 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T15:30:26-0800] [MainThread] [I] [toil.leader] Issued job 'run_indel_caller' kind-run_indel_caller/instance-0_hyd44y with job batch system ID: 51 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T15:30:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpzqz6af18/worker_log.txt +[2020-11-27T15:30:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpf0y5ll2z/worker_log.txt +[2020-11-27T15:30:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7rrslj8h/worker_log.txt +[2020-11-27T15:30:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgmiv6whh/worker_log.txt +[2020-11-27T15:30:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgyqfpaci/worker_log.txt +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-wlu1vdpy +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-sydh7m88 with job batch system ID: 52 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-xmq5os92 with job batch system ID: 53 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-bj0twc34 +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-yzx302yq with job batch system ID: 54 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-dkq5pakk with job batch system ID: 55 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-npf1btg7 with job batch system ID: 56 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-f7d6aq5_ with job batch system ID: 57 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-29o5kgl4 with job batch system ID: 58 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-yjn16wtm with job batch system ID: 59 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-dz5pfkoc with job batch system ID: 60 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-wlvnhdem with job batch system ID: 61 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-kc69f8vr with job batch system ID: 62 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-l7vpzyz4 with job batch system ID: 63 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-e3gki4ix with job batch system ID: 64 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-l0norp0m with job batch system ID: 65 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-z1dbqpwz with job batch system ID: 66 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-rnhvgo7a with job batch system ID: 67 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-pihl7nio with job batch system ID: 68 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-1zw81rwj with job batch system ID: 69 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-t670_m75 with job batch system ID: 70 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-rlk8veoh with job batch system ID: 71 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-ei8t4y9l with job batch system ID: 72 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-7an9pegh with job batch system ID: 73 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-6xxfkfl2 with job batch system ID: 74 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-030qvgzd with job batch system ID: 75 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-_90_leu0 with job batch system ID: 76 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-f_o8awfa with job batch system ID: 77 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-7kwh6teo with job batch system ID: 78 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-qi2fqyz_ +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-b726_qzy with job batch system ID: 79 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-1pmpwd12 with job batch system ID: 80 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-7odvop3a with job batch system ID: 81 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-p33vv7px with job batch system ID: 82 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-_f8b0j_s with job batch system ID: 83 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-48j7gqm7 with job batch system ID: 84 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-gwas7gvl with job batch system ID: 85 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-gj7djl5i with job batch system ID: 86 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-e1l03y4y with job batch system ID: 87 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-ppvdu8e6 with job batch system ID: 88 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-rae7ba_d with job batch system ID: 89 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-t85wxba1 with job batch system ID: 90 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-vpr5swji with job batch system ID: 91 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-k1_0bq77 with job batch system ID: 92 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-_iy3ws7d with job batch system ID: 93 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-js_02rcy with job batch system ID: 94 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-oz17e__0 with job batch system ID: 95 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-_d6jcuz2 with job batch system ID: 96 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-nge1n1b6 with job batch system ID: 97 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-sm40dvu4 with job batch system ID: 98 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-w2jq7q7f with job batch system ID: 99 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-scovrnh_ with job batch system ID: 100 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-11hbfabv with job batch system ID: 101 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-jogi7gny with job batch system ID: 102 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-awq_3zbw with job batch system ID: 103 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:30:34-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:30:34: INDELs are currently unsupported.... Skipping. +[2020-11-27T15:30:34-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:30:34: Ran INDEL on TEST successfully +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-ixd9le0x +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Issued job 'run_strelka_full' kind-run_strelka_full/instance-eyjhnv7s with job batch system ID: 104 and cores: 25, disk: 6.0 G, and memory: 6.0 G +[2020-11-27T15:30:34-0800] [MainThread] [I] [toil.leader] Job ended: 'run_indel_caller' kind-run_indel_caller/instance-0_hyd44y +[2020-11-27T15:30:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpesa2so7z/worker_log.txt +[2020-11-27T15:30:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpu92a42dn/worker_log.txt +[2020-11-27T15:30:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpsm6r3r5b/worker_log.txt +[2020-11-27T15:30:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpsuwyrgsp/worker_log.txt +[2020-11-27T15:30:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpzce98ovj/worker_log.txt +[2020-11-27T15:30:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp3opzklji/worker_log.txt +[2020-11-27T15:30:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpkq8mzsos/worker_log.txt +[2020-11-27T15:30:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp27jaot_a/worker_log.txt +[2020-11-27T15:30:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpxktcgy0l/worker_log.txt +[2020-11-27T15:30:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpski0_aoh/worker_log.txt +[2020-11-27T15:30:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp8jksmzjh/worker_log.txt +[2020-11-27T15:30:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7s7n07j6/worker_log.txt +[2020-11-27T15:30:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp4_ca10fw/worker_log.txt +[2020-11-27T15:30:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpjssvt2um/worker_log.txt +[2020-11-27T15:30:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpda7aa7uv/worker_log.txt +[2020-11-27T15:30:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_4f7y4xf/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdexd8dz3/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpqwwm78p5/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp03737i7d/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbqniaq08/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmphookx11o/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpq7vm20uo/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpzop2s49g/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpnihjfxly/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpzss_uf2j/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmps7dynpuv/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpmvttq84m/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpyrc7c1c5/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpi9k0vssj/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpvg6r6l1h/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpsrnjqocv/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpenizrkpt/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpuntd95um/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpv4lm1h2p/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_gk_5ihd/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpp5bnguf7/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpkz1pcdwk/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpigar51du/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpc_ano1qu/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpt47__tk4/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmplbezn3p5/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpp29m9n3e/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp33ku_2vl/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmprzjuhpl2/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpm29c2rti/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpcysfxzeh/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmperqta3p0/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmplkcjsp10/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7a1yq49m/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpmid1dqi2/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp790fzmyi/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpknrpwu5l/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp050f17ib/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-xmq5os92 +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_pileup' kind-run_pileup/instance-ll3vci4r with job batch system ID: 105 and cores: 1, disk: 4.8 G, and memory: 6.0 G +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpygcr5uic/worker_log.txt +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-_iy3ws7d +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-94opbfzg with job batch system ID: 106 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-sydh7m88 +[2020-11-27T15:30:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_somaticsniper_full' kind-run_somaticsniper_full/instance-bljmwy9w with job batch system ID: 107 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-f7d6aq5_ +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fgj0w2_w with job batch system ID: 108 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-dkq5pakk +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-t15qgjy0 with job batch system ID: 109 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-npf1btg7 +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-989tgher with job batch system ID: 110 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-yjn16wtm +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-4f2ha64d with job batch system ID: 111 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-yzx302yq +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5zdtbtsn with job batch system ID: 112 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-awq_3zbw +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-mrt1ff_h with job batch system ID: 113 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-l7vpzyz4 +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r0a3rxak with job batch system ID: 114 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-29o5kgl4 +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ey78_apt with job batch system ID: 115 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-l0norp0m +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h0tdg49o with job batch system ID: 116 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-dz5pfkoc +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-1jh6nqug with job batch system ID: 117 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-e3gki4ix +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9i484bqi with job batch system ID: 118 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-z1dbqpwz +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-hxsjhjvp with job batch system ID: 119 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-b726_qzy +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-6pvv69ks with job batch system ID: 120 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-rlk8veoh +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5i9__ll1 with job batch system ID: 121 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-rae7ba_d +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-bkhwp755 with job batch system ID: 122 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-sm40dvu4 +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-_k_eemal with job batch system ID: 123 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-rnhvgo7a +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ntmj9aw5 with job batch system ID: 124 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-pihl7nio +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-e1yejtsm with job batch system ID: 125 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-7odvop3a +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-urjm7ey8 with job batch system ID: 126 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-kc69f8vr +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-f3zgk1bv with job batch system ID: 127 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-ei8t4y9l +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mgq6ru10 with job batch system ID: 128 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-6xxfkfl2 +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-77hux1xy with job batch system ID: 129 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-p33vv7px +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-s4yqjm54 with job batch system ID: 130 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-_f8b0j_s +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-4qoiyflm with job batch system ID: 131 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-gwas7gvl +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-7axkvh07 with job batch system ID: 132 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-1zw81rwj +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-_u307ivu with job batch system ID: 133 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-7an9pegh +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-p8mcfs96 with job batch system ID: 134 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-t670_m75 +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-du24kqrt with job batch system ID: 135 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmprvcrg6zf/worker_log.txt +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-wlvnhdem +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-y2fcywgt with job batch system ID: 136 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-030qvgzd +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-m6rpu3gh with job batch system ID: 137 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-_90_leu0 +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gyttywxo with job batch system ID: 138 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-f_o8awfa +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jvmeok8w with job batch system ID: 139 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-7kwh6teo +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xqryz_ub with job batch system ID: 140 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-1pmpwd12 +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-tw5yau0c with job batch system ID: 141 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-48j7gqm7 +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-065d2_jn with job batch system ID: 142 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-gj7djl5i +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-h42fw8gb with job batch system ID: 143 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-e1l03y4y +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-syxzhyb3 with job batch system ID: 144 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-ppvdu8e6 +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-to1xdt8b with job batch system ID: 145 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-t85wxba1 +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-bu3d_jna with job batch system ID: 146 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpwz1oysdh/worker_log.txt +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-vpr5swji +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-4f0h4ksc with job batch system ID: 147 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-k1_0bq77 +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-je31_viw with job batch system ID: 148 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-js_02rcy +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-3wci2pk1 with job batch system ID: 149 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-oz17e__0 +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-epn0ookc with job batch system ID: 150 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-_d6jcuz2 +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-83qdvxtt with job batch system ID: 151 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-nge1n1b6 +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-sza9a3st with job batch system ID: 152 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-w2jq7q7f +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-ywo_o_vy with job batch system ID: 153 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-scovrnh_ +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-yln073f1 with job batch system ID: 154 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-11hbfabv +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-itr0gvem with job batch system ID: 155 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1erdb6o4/worker_log.txt +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-jogi7gny +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-run_muse_perchrom/instance-lqf3jcod with job batch system ID: 156 and cores: 1, disk: 5.1 G, and memory: 6.0 G +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpohy4lr4g/worker_log.txt +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpmy1f1o_b/worker_log.txt +[2020-11-27T15:30:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpd27f5zl0/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpp3f1aq8c/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmph1kic21q/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpnmdolwtx/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1ksntmcb/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpmc2pa6l5/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmphcxk6l2f/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_ebd9oa9/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpe1wue9cp/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpszgez8hv/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp8hsxw54r/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpwbirp4dt/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpx6qdq3uh/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7wgwzusc/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp2j2p3gfc/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpxs223ac_/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp8vaabpem/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp6dritv6b/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpikfkfby0/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpqcs2xmro/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpcgoqqm_k/worker_log.txt +[2020-11-27T15:30:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpd4hj4hse/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp8v4mm9jm/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpifqjn_2v/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp64rfvlc6/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpce2oljf3/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpwtkkuiep/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp4b12h10r/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpfnprwuxz/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp95tt8q1t/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgtkri8ub/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpuruf2bw4/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_lhfh7li/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_s4uadsq/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpl28na2i7/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpsog8xotm/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpfd39u5qj/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpqyg6n15k/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpksld4p2s/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1o66g3ve/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpqz0ug0h8/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpey61bb2g/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp4z9_dxvw/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpzan3baq_/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpwjq014mx/worker_log.txt +[2020-11-27T15:30:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp9sxdiitc/worker_log.txt +[2020-11-27T15:51:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:51:55: Ran MuSE on TEST:chrM successfully +[2020-11-27T15:51:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:51:55: Ran MuSE on TEST:chrY successfully +[2020-11-27T15:51:55-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-mrt1ff_h +[2020-11-27T15:51:55-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-xv97ob90 with job batch system ID: 157 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:51:56-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-lqf3jcod +[2020-11-27T15:51:56-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-w3609gux with job batch system ID: 158 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:51:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpq8gpmtvh/worker_log.txt +[2020-11-27T15:51:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpp4ui8paw/worker_log.txt +[2020-11-27T15:51:57-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-w3609gux +[2020-11-27T15:51:57-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-zkwp8_27 with job batch system ID: 159 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:51:57-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-xv97ob90 +[2020-11-27T15:51:57-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-fova29au with job batch system ID: 160 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:51:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:51:57: Ran MuSE on TEST:chr22 successfully +[2020-11-27T15:51:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpi2t21uhy/worker_log.txt +[2020-11-27T15:51:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpxlzeugkc/worker_log.txt +[2020-11-27T15:51:58-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-yln073f1 +[2020-11-27T15:51:58-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-pkbplyf6 with job batch system ID: 161 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:51:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpm09em29f/worker_log.txt +[2020-11-27T15:51:59-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-pkbplyf6 +[2020-11-27T15:51:59-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-nl0_ti7j with job batch system ID: 162 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1za1pzod/worker_log.txt +[2020-11-27T15:52:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:01: Ran MuSE on TEST:chr20 successfully +[2020-11-27T15:52:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-_k_eemal +[2020-11-27T15:52:02-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-owb13we1 with job batch system ID: 163 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp6hmkl32k/worker_log.txt +[2020-11-27T15:52:03-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-owb13we1 +[2020-11-27T15:52:03-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-ou8cav58 with job batch system ID: 164 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:04-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp0m2_kh78/worker_log.txt +[2020-11-27T15:52:08-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:08: Ran MuSE on TEST:chr15 successfully +[2020-11-27T15:52:09-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-94opbfzg +[2020-11-27T15:52:09-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-6xkj6w56 with job batch system ID: 165 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:10-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpyb0qj11f/worker_log.txt +[2020-11-27T15:52:10-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-6xkj6w56 +[2020-11-27T15:52:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-grkfkjp4 with job batch system ID: 166 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpew8jfz5b/worker_log.txt +[2020-11-27T15:52:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:26: Ran MuSE on TEST:chr21 successfully +[2020-11-27T15:52:27-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-ywo_o_vy +[2020-11-27T15:52:27-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-md4dzb30 with job batch system ID: 167 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:27-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:27: Ran MuSE on TEST:chrX successfully +[2020-11-27T15:52:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:28: Ran MuSE on TEST:chr17 successfully +[2020-11-27T15:52:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:28: Ran MuSE on TEST:chr18 successfully +[2020-11-27T15:52:28-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp6e0322dy/worker_log.txt +[2020-11-27T15:52:28-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-itr0gvem +[2020-11-27T15:52:28-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-1vt69zfh with job batch system ID: 168 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:28-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-md4dzb30 +[2020-11-27T15:52:28-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-x3c2w82r with job batch system ID: 169 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:28-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-epn0ookc +[2020-11-27T15:52:28-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-dc5e9kpa with job batch system ID: 170 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:29-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-83qdvxtt +[2020-11-27T15:52:29-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-moesufn2 with job batch system ID: 171 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:29-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:29: Ran MuSE on TEST:chr16 successfully +[2020-11-27T15:52:29-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:29: Ran MuSE on TEST:chr6 successfully +[2020-11-27T15:52:29-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpj_et4kfw/worker_log.txt +[2020-11-27T15:52:29-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpa0hjuwdu/worker_log.txt +[2020-11-27T15:52:29-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:29: Ran MuSE on TEST:chr19 successfully +[2020-11-27T15:52:29-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpl5hjepfr/worker_log.txt +[2020-11-27T15:52:29-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-1vt69zfh +[2020-11-27T15:52:29-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-e_blh1z0 with job batch system ID: 172 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:29-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmptzul1d3d/worker_log.txt +[2020-11-27T15:52:30-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-3wci2pk1 +[2020-11-27T15:52:30-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-_ryylcz1 with job batch system ID: 173 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:30-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-dc5e9kpa +[2020-11-27T15:52:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-0gl6k6z2 with job batch system ID: 174 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:30-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-moesufn2 +[2020-11-27T15:52:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-8jgfcelw with job batch system ID: 175 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:30-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-065d2_jn +[2020-11-27T15:52:30-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-r58gvchk with job batch system ID: 176 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:30-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-sza9a3st +[2020-11-27T15:52:30-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-k1x33g_e with job batch system ID: 177 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgcv1wmvw/worker_log.txt +[2020-11-27T15:52:30-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:30: Ran MuSE on TEST:chr13 successfully +[2020-11-27T15:52:30-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:30: Ran MuSE on TEST:chr9 successfully +[2020-11-27T15:52:30-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:30: Ran MuSE on TEST:chr14 successfully +[2020-11-27T15:52:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpzgo7dgv_/worker_log.txt +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpancs0lkt/worker_log.txt +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpky3tie01/worker_log.txt +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpm7k0ufiz/worker_log.txt +[2020-11-27T15:52:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:31: Ran MuSE on TEST:chr8 successfully +[2020-11-27T15:52:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:31: Ran MuSE on TEST:chr1 successfully +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-_ryylcz1 +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-et_09yzn with job batch system ID: 178 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpd9_5vlle/worker_log.txt +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-r58gvchk +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-o5r4pz2v with job batch system ID: 179 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-4f0h4ksc +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-2c_bpy8a with job batch system ID: 180 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-k1x33g_e +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-ir_1nxzg with job batch system ID: 181 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-syxzhyb3 +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-_5cjudo0 with job batch system ID: 182 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-je31_viw +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-2c71_1fe with job batch system ID: 183 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-h42fw8gb +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-5clqy0kx with job batch system ID: 184 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-6pvv69ks +[2020-11-27T15:52:31-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-fu4wl7y_ with job batch system ID: 185 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpvwzpyv4f/worker_log.txt +[2020-11-27T15:52:32-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:32: Ran MuSE on TEST:chr11 successfully +[2020-11-27T15:52:32-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:32: Ran MuSE on TEST:chr5 successfully +[2020-11-27T15:52:32-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:32: Ran MuSE on TEST:chr3 successfully +[2020-11-27T15:52:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpz8xwrzf6/worker_log.txt +[2020-11-27T15:52:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdeq1jegk/worker_log.txt +[2020-11-27T15:52:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp30xoekpe/worker_log.txt +[2020-11-27T15:52:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpxfm8qfy5/worker_log.txt +[2020-11-27T15:52:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmph3xqfqtn/worker_log.txt +[2020-11-27T15:52:32-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-2c_bpy8a +[2020-11-27T15:52:32-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-lerujkui with job batch system ID: 186 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpg938603j/worker_log.txt +[2020-11-27T15:52:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpu6bfd78z/worker_log.txt +[2020-11-27T15:52:32-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:32: Ran MuSE on TEST:chr10 successfully +[2020-11-27T15:52:32-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-_5cjudo0 +[2020-11-27T15:52:32-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-mmlo7lhq with job batch system ID: 187 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:32-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-2c71_1fe +[2020-11-27T15:52:32-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-saf6qn11 with job batch system ID: 188 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:33-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-fu4wl7y_ +[2020-11-27T15:52:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-ngpptjca with job batch system ID: 189 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:33-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-urjm7ey8 +[2020-11-27T15:52:33-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-gz2vw7kn with job batch system ID: 190 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:33-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-5clqy0kx +[2020-11-27T15:52:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-bznkc2va with job batch system ID: 191 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:33-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-4qoiyflm +[2020-11-27T15:52:33-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-uryds7kk with job batch system ID: 192 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:33-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-bkhwp755 +[2020-11-27T15:52:33-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-yivat4a_ with job batch system ID: 193 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp4nqz7n9r/worker_log.txt +[2020-11-27T15:52:33-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-to1xdt8b +[2020-11-27T15:52:33-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-nrmj7qhd with job batch system ID: 194 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:33: Ran MuSE on TEST:chr7 successfully +[2020-11-27T15:52:33-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:33: Ran MuSE on TEST:chr12 successfully +[2020-11-27T15:52:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmprsopqnbp/worker_log.txt +[2020-11-27T15:52:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp257_360n/worker_log.txt +[2020-11-27T15:52:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp0tyon6xv/worker_log.txt +[2020-11-27T15:52:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpo59sfe5u/worker_log.txt +[2020-11-27T15:52:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp9t5ajayu/worker_log.txt +[2020-11-27T15:52:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpo4amx4h0/worker_log.txt +[2020-11-27T15:52:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp74hx_vjc/worker_log.txt +[2020-11-27T15:52:34-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:34: Ran MuSE on TEST:chr4 successfully +[2020-11-27T15:52:34-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:34: Ran MuSE on TEST:chr2 successfully +[2020-11-27T15:52:34-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-gz2vw7kn +[2020-11-27T15:52:34-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-xxxjj6dq with job batch system ID: 195 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpkv12bsha/worker_log.txt +[2020-11-27T15:52:34-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-yivat4a_ +[2020-11-27T15:52:34-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-viiei36o with job batch system ID: 196 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:34-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-uryds7kk +[2020-11-27T15:52:34-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-1fpfkwaz with job batch system ID: 197 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:34-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-nrmj7qhd +[2020-11-27T15:52:34-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-uh2t4lnf with job batch system ID: 198 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:34-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-7axkvh07 +[2020-11-27T15:52:34-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-hci2aequ with job batch system ID: 199 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:34-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-bu3d_jna +[2020-11-27T15:52:34-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-z505s6ae with job batch system ID: 200 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:34-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-tw5yau0c +[2020-11-27T15:52:34-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-s458y9ti with job batch system ID: 201 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpa99di868/worker_log.txt +[2020-11-27T15:52:35-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-run_muse_perchrom/instance-s4yqjm54 +[2020-11-27T15:52:35-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ga_m7kic with job batch system ID: 202 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpv_c2hcg5/worker_log.txt +[2020-11-27T15:52:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp18470_si/worker_log.txt +[2020-11-27T15:52:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpr8adp3rn/worker_log.txt +[2020-11-27T15:52:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpxgcnc0xb/worker_log.txt +[2020-11-27T15:52:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp0_iejo8j/worker_log.txt +[2020-11-27T15:52:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpwylnexyd/worker_log.txt +[2020-11-27T15:52:35-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-hci2aequ +[2020-11-27T15:52:35-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-nx20zj44 with job batch system ID: 203 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:36-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-z505s6ae +[2020-11-27T15:52:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-zhreqges with job batch system ID: 204 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:36-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-s458y9ti +[2020-11-27T15:52:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-ovjiaqxc with job batch system ID: 205 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpl2sq0hnd/worker_log.txt +[2020-11-27T15:52:36-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ga_m7kic +[2020-11-27T15:52:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-japuqut7 with job batch system ID: 206 and cores: 1, disk: 2.9 G, and memory: 6.0 G +[2020-11-27T15:52:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpe1lmr4ab/worker_log.txt +[2020-11-27T15:52:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbs6rxa5r/worker_log.txt +[2020-11-27T15:52:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpsaost95o/worker_log.txt +[2020-11-27T15:52:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpzavv3l1l/worker_log.txt +[2020-11-27T15:52:37-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-zkwp8_27 +[2020-11-27T15:52:37-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-epy5tdv0 with job batch system ID: 207 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:37: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-zkwp8_27/file-sj68xiyb/chrY.vcf to output location +[2020-11-27T15:52:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:37: Ran MuSE sump on TEST:chrY successfully +[2020-11-27T15:52:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:37: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-ir_1nxzg/file-ceezth9u/chr19.vcf to output location +[2020-11-27T15:52:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:37: Ran MuSE sump on TEST:chr19 successfully +[2020-11-27T15:52:37-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-ir_1nxzg +[2020-11-27T15:52:37-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-b961bbif with job batch system ID: 208 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:37-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-et_09yzn +[2020-11-27T15:52:37-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-rbp3d4jp with job batch system ID: 209 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:38: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-e_blh1z0/file-3423c4ja/chrX.vcf to output location +[2020-11-27T15:52:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:38: Ran MuSE sump on TEST:chrX successfully +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-0gl6k6z2 +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-8xodvm8f with job batch system ID: 210 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:38: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-et_09yzn/file-malzusd7/chr16.vcf to output location +[2020-11-27T15:52:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:38: Ran MuSE sump on TEST:chr16 successfully +[2020-11-27T15:52:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:38: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-0gl6k6z2/file-0yuok_tx/chr17.vcf to output location +[2020-11-27T15:52:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:38: Ran MuSE sump on TEST:chr17 successfully +[2020-11-27T15:52:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:38: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-grkfkjp4/file-sf526ypy/chr15.vcf to output location +[2020-11-27T15:52:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:38: Ran MuSE sump on TEST:chr15 successfully +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-grkfkjp4 +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-uhupgud6 with job batch system ID: 211 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-e_blh1z0 +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-4zozy4rg with job batch system ID: 212 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-fova29au +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-09s5q8ec with job batch system ID: 213 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-nl0_ti7j +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-hwyar0k2 with job batch system ID: 214 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmphar40u2j/worker_log.txt +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-o5r4pz2v +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-l5n1oo0i with job batch system ID: 215 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpg4be8mba/worker_log.txt +[2020-11-27T15:52:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:38: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-fova29au/file-o145653e/chrM.vcf to output location +[2020-11-27T15:52:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:38: Ran MuSE sump on TEST:chrM successfully +[2020-11-27T15:52:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:38: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-o5r4pz2v/file-0qbexcy2/chr6.vcf to output location +[2020-11-27T15:52:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:38: Ran MuSE sump on TEST:chr6 successfully +[2020-11-27T15:52:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:38: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-ou8cav58/file-mm9t2_j4/chr20.vcf to output location +[2020-11-27T15:52:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:38: Ran MuSE sump on TEST:chr20 successfully +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-ou8cav58 +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-wsulneug with job batch system ID: 216 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:38: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-nl0_ti7j/file-f51b7wy6/chr22.vcf to output location +[2020-11-27T15:52:38-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:38: Ran MuSE sump on TEST:chr22 successfully +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-epy5tdv0 +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-w3609gux with job batch system ID: 217 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-b961bbif +[2020-11-27T15:52:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-k1x33g_e with job batch system ID: 218 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-x3c2w82r +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-w78a0t5w with job batch system ID: 219 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbcn0d2ga/worker_log.txt +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-8jgfcelw +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-tmgexsbr with job batch system ID: 220 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-rbp3d4jp +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-_ryylcz1 with job batch system ID: 221 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:39: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-x3c2w82r/file-o4uu3rpx/chr21.vcf to output location +[2020-11-27T15:52:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:39: Ran MuSE sump on TEST:chr21 successfully +[2020-11-27T15:52:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:39: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-8jgfcelw/file-w7e0ubcb/chr18.vcf to output location +[2020-11-27T15:52:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:39: Ran MuSE sump on TEST:chr18 successfully +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpn_gql1gm/worker_log.txt +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgvedjsgq/worker_log.txt +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpmqm6ahx2/worker_log.txt +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-uhupgud6 +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-6xkj6w56 with job batch system ID: 222 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-8xodvm8f +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-dc5e9kpa with job batch system ID: 223 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpu1wn62m0/worker_log.txt +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp8k_te373/worker_log.txt +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-4zozy4rg +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-1vt69zfh with job batch system ID: 224 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-hwyar0k2 +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-pkbplyf6 with job batch system ID: 225 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp43bgx1kn/worker_log.txt +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpq0if7wyx/worker_log.txt +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpyrz13tof/worker_log.txt +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-09s5q8ec +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-xv97ob90 with job batch system ID: 226 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-w3609gux +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-jogi7gny with job batch system ID: 227 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmprwowewu1/worker_log.txt +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp26e1ht7z/worker_log.txt +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1l2s_8gj/worker_log.txt +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-l5n1oo0i +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-r58gvchk with job batch system ID: 228 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-k1x33g_e +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-nge1n1b6 with job batch system ID: 229 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-wsulneug +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-owb13we1 with job batch system ID: 230 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-w78a0t5w +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-md4dzb30 with job batch system ID: 231 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-tmgexsbr +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-moesufn2 with job batch system ID: 232 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp0tgdxee9/worker_log.txt +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp8bmpveaf/worker_log.txt +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpizm6tdgz/worker_log.txt +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-_ryylcz1 +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-js_02rcy with job batch system ID: 233 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-6xkj6w56 +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-_iy3ws7d with job batch system ID: 234 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-dc5e9kpa +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-oz17e__0 with job batch system ID: 235 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmptnt1ni8g/worker_log.txt +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpps_yk5a_/worker_log.txt +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-pkbplyf6 +[2020-11-27T15:52:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-scovrnh_ with job batch system ID: 236 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-1vt69zfh +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-11hbfabv with job batch system ID: 237 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpde5f3zi1/worker_log.txt +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp22ix5kr9/worker_log.txt +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-bznkc2va +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-o1rk5ltv with job batch system ID: 238 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-xv97ob90 +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-awq_3zbw with job batch system ID: 239 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-jogi7gny +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpjz_ihn2a/worker_log.txt +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-r58gvchk +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-48j7gqm7 with job batch system ID: 240 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-lerujkui +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-bzfq_qu_ with job batch system ID: 241 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-mmlo7lhq +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ng3_9nvg with job batch system ID: 242 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:41: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-mmlo7lhq/file-avvxz0co/chr9.vcf to output location +[2020-11-27T15:52:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:41: Ran MuSE sump on TEST:chr9 successfully +[2020-11-27T15:52:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:41: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-bznkc2va/file-rrt8v6k4/chr8.vcf to output location +[2020-11-27T15:52:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:41: Ran MuSE sump on TEST:chr8 successfully +[2020-11-27T15:52:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:41: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-lerujkui/file-qns1a_fx/chr13.vcf to output location +[2020-11-27T15:52:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:41: Ran MuSE sump on TEST:chr13 successfully +[2020-11-27T15:52:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:41: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-saf6qn11/file-9fxbsjf9/chr14.vcf to output location +[2020-11-27T15:52:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:41: Ran MuSE sump on TEST:chr14 successfully +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-saf6qn11 +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-o62ll4pj with job batch system ID: 243 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpm70ma553/worker_log.txt +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpxv08scfh/worker_log.txt +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-nge1n1b6 +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp173otdlo/worker_log.txt +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-owb13we1 +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-sm40dvu4 with job batch system ID: 244 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7z7x4d90/worker_log.txt +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-ngpptjca +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-_bnn9cpx with job batch system ID: 245 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-md4dzb30 +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-w2jq7q7f with job batch system ID: 246 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-uh2t4lnf +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-98wlklk3 with job batch system ID: 247 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-viiei36o +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ct3dlqck with job batch system ID: 248 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-moesufn2 +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-_d6jcuz2 with job batch system ID: 249 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-1fpfkwaz +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-klr0lb3k with job batch system ID: 250 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpnjzuem5n/worker_log.txt +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpf0w7oq94/worker_log.txt +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpolpsxugf/worker_log.txt +[2020-11-27T15:52:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:41: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-uh2t4lnf/file-4yrcf9rw/chr10.vcf to output location +[2020-11-27T15:52:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:41: Ran MuSE sump on TEST:chr10 successfully +[2020-11-27T15:52:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:41: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-ngpptjca/file-72xwd_aj/chr1.vcf to output location +[2020-11-27T15:52:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:41: Ran MuSE sump on TEST:chr1 successfully +[2020-11-27T15:52:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:41: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-viiei36o/file-9o63l6jx/chr11.vcf to output location +[2020-11-27T15:52:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:41: Ran MuSE sump on TEST:chr11 successfully +[2020-11-27T15:52:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:41: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-1fpfkwaz/file-n8c5ljqi/chr5.vcf to output location +[2020-11-27T15:52:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:41: Ran MuSE sump on TEST:chr5 successfully +[2020-11-27T15:52:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:41: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-xxxjj6dq/file-pp3ogvqg/chr3.vcf to output location +[2020-11-27T15:52:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:41: Ran MuSE sump on TEST:chr3 successfully +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-_iy3ws7d +[2020-11-27T15:52:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-js_02rcy +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-xxxjj6dq +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-h8nl55ox with job batch system ID: 251 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-oz17e__0 +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_syxasdf/worker_log.txt +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-scovrnh_ +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpldza7uo3/worker_log.txt +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmplbnchoem/worker_log.txt +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpfv2f21ud/worker_log.txt +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpd6wumdkp/worker_log.txt +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpqw_lwg6a/worker_log.txt +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-11hbfabv +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmps5g3_dqi/worker_log.txt +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-ovjiaqxc +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-8un8rmqt with job batch system ID: 252 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-o1rk5ltv +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-5clqy0kx with job batch system ID: 253 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-nx20zj44 +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-1knzgrxv with job batch system ID: 254 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-zhreqges +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ujmd46jc with job batch system ID: 255 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-awq_3zbw +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-48j7gqm7 +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpokvrqxvk/worker_log.txt +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-o62ll4pj +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-2c71_1fe with job batch system ID: 256 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpmqbzk25w/worker_log.txt +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ng3_9nvg +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-_5cjudo0 with job batch system ID: 257 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp84_7dj3g/worker_log.txt +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpcnmqoegc/worker_log.txt +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp543esq8e/worker_log.txt +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmptszmcicz/worker_log.txt +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-bzfq_qu_ +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-2c_bpy8a with job batch system ID: 258 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-sm40dvu4 +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-w2jq7q7f +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ct3dlqck +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-yivat4a_ with job batch system ID: 259 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:42: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-ovjiaqxc/file-jxmln1ub/chr2.vcf to output location +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpz06b8qv_/worker_log.txt +[2020-11-27T15:52:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:42: Ran MuSE sump on TEST:chr2 successfully +[2020-11-27T15:52:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:42: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-zhreqges/file-m4c2nn5w/chr12.vcf to output location +[2020-11-27T15:52:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:42: Ran MuSE sump on TEST:chr12 successfully +[2020-11-27T15:52:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:42: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-nx20zj44/file-bqrx74fk/chr7.vcf to output location +[2020-11-27T15:52:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:42: Ran MuSE sump on TEST:chr7 successfully +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-98wlklk3 +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-nrmj7qhd with job batch system ID: 260 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpior89iip/worker_log.txt +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-_bnn9cpx +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-fu4wl7y_ with job batch system ID: 261 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpvl3glmsz/worker_log.txt +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-_d6jcuz2 +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-run_muse_sump_perchrom/instance-japuqut7 +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ilzerk5f with job batch system ID: 262 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-klr0lb3k +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-uryds7kk with job batch system ID: 263 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-h8nl55ox +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-gz2vw7kn with job batch system ID: 264 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:43: Exporting files/for-job/kind-run_muse_sump_perchrom/instance-japuqut7/file-3wej0uda/chr4.vcf to output location +[2020-11-27T15:52:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:43: Ran MuSE sump on TEST:chr4 successfully +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpkl02ktp7/worker_log.txt +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_6vl78_k/worker_log.txt +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpz5154tb1/worker_log.txt +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpydycm7q9/worker_log.txt +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-5clqy0kx +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-gj7djl5i with job batch system ID: 265 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ujmd46jc +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-z505s6ae with job batch system ID: 266 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-8un8rmqt +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-s458y9ti with job batch system ID: 267 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpm3qmp62e/worker_log.txt +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-1knzgrxv +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-hci2aequ with job batch system ID: 268 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1so2w0ax/worker_log.txt +[2020-11-27T15:52:43-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-_5cjudo0 +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-e1l03y4y with job batch system ID: 269 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-2c71_1fe +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-k1_0bq77 with job batch system ID: 270 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdn1rzptr/worker_log.txt +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmphodmxyq2/worker_log.txt +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmph40lvx97/worker_log.txt +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-2c_bpy8a +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-vpr5swji with job batch system ID: 271 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbwaehibd/worker_log.txt +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-fu4wl7y_ +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-b726_qzy with job batch system ID: 272 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-yivat4a_ +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-rae7ba_d with job batch system ID: 273 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp40qgqmat/worker_log.txt +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-nrmj7qhd +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-ppvdu8e6 with job batch system ID: 274 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmppmkha965/worker_log.txt +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1jfqanri/worker_log.txt +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ilzerk5f +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_sump_perchrom' kind-Job/instance-ga_m7kic with job batch system ID: 275 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-gz2vw7kn +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-7odvop3a with job batch system ID: 276 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-uryds7kk +[2020-11-27T15:52:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-_f8b0j_s with job batch system ID: 277 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp11j33d20/worker_log.txt +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpws24d8pz/worker_log.txt +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-z505s6ae +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-t85wxba1 with job batch system ID: 278 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp8dw48vg_/worker_log.txt +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpk9qq5nx0/worker_log.txt +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp28lrk8um/worker_log.txt +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-s458y9ti +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-1pmpwd12 with job batch system ID: 279 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp8dbd2sbs/worker_log.txt +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-gj7djl5i +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-hci2aequ +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-gwas7gvl with job batch system ID: 280 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpejx7lskr/worker_log.txt +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-e1l03y4y +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpb6a41tlj/worker_log.txt +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-k1_0bq77 +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-vpr5swji +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp9uq9jzn7/worker_log.txt +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-b726_qzy +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpb_uq8hts/worker_log.txt +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-ppvdu8e6 +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-rae7ba_d +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7uumfwh3/worker_log.txt +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp4afhebj5/worker_log.txt +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp2u1mwwj2/worker_log.txt +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-7odvop3a +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_sump_perchrom' kind-Job/instance-ga_m7kic +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse_perchrom' kind-EncapsulatedJob/instance-p33vv7px with job batch system ID: 281 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:52:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-_f8b0j_s +[2020-11-27T15:52:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpv09pmisp/worker_log.txt +[2020-11-27T15:52:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpm26l6t8s/worker_log.txt +[2020-11-27T15:52:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-1pmpwd12 +[2020-11-27T15:52:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp8v05i45m/worker_log.txt +[2020-11-27T15:52:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-t85wxba1 +[2020-11-27T15:52:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-gwas7gvl +[2020-11-27T15:52:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp4c0n4fzu/worker_log.txt +[2020-11-27T15:52:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse_perchrom' kind-EncapsulatedJob/instance-p33vv7px +[2020-11-27T15:52:46-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ccadqh9a with job batch system ID: 282 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpo31joczm/worker_log.txt +[2020-11-27T15:52:47-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ccadqh9a +[2020-11-27T15:52:51-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:52:51: Ran samtools pileup on TEST successfully +[2020-11-27T15:52:52-0800] [MainThread] [I] [toil.leader] Job ended: 'run_pileup' kind-run_pileup/instance-ll3vci4r +[2020-11-27T15:52:52-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-p6aqi182 with job batch system ID: 283 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:52:52-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpw51s0x5i/worker_log.txt +[2020-11-27T15:52:52-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-p6aqi182 +[2020-11-27T15:54:13-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:54:13: Ran SomaticSniper on TEST successfully +[2020-11-27T15:54:14-0800] [MainThread] [I] [toil.leader] Job ended: 'run_somaticsniper_full' kind-run_somaticsniper_full/instance-bljmwy9w +[2020-11-27T15:54:14-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-q74ncvyg with job batch system ID: 284 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:54:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpwiulj79k/worker_log.txt +[2020-11-27T15:54:15-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-q74ncvyg +[2020-11-27T15:54:15-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-u3pcg5lk with job batch system ID: 285 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:54:15-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpjph0f42s/worker_log.txt +[2020-11-27T15:54:16-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-u3pcg5lk +[2020-11-27T15:54:16-0800] [MainThread] [I] [toil.leader] Issued job 'filter_somaticsniper' kind-filter_somaticsniper/instance-6mjhihuo with job batch system ID: 286 and cores: 1, disk: 4.8 G, and memory: 6.0 G +[2020-11-27T15:54:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp74llme32/worker_log.txt +[2020-11-27T15:55:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:01: Ran strelka on TEST successfully +[2020-11-27T15:55:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_strelka_full' kind-run_strelka_full/instance-eyjhnv7s +[2020-11-27T15:55:02-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-fx0_pxis with job batch system ID: 287 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:55:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpzaz9n7xq/worker_log.txt +[2020-11-27T15:55:03-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-fx0_pxis +[2020-11-27T15:55:03-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-gi0_24pa with job batch system ID: 288 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T15:55:04-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpx5i34brd/worker_log.txt +[2020-11-27T15:55:04-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-gi0_24pa +[2020-11-27T15:55:04-0800] [MainThread] [I] [toil.leader] Issued job 'unmerge' kind-unmerge/instance-e8xvdfut with job batch system ID: 289 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T15:55:04-0800] [MainThread] [I] [toil.leader] Issued job 'unmerge' kind-unmerge/instance-n0yye4h5 with job batch system ID: 290 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T15:55:04-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmppy_h37ns/worker_log.txt +[2020-11-27T15:55:04-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpd3rbacj9/worker_log.txt +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-bc230vn2/chr3.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-a49zk1n6/chrM.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-_3uv_bdx/chr11.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-_jlzpmk1/chr19.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-kz4htesk/chrX.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-r1ttkdgc/chr13.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-xalgznt2/chr1.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-0n6n9kf7/chr18.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-7w0snnun/chr20.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-0oa_wn1_/chrY.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-sdhcydu3/chr8.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-25vnscow/chr14.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-zfpr8mg9/chr21.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-o38b63y5/chr6.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-r9i2y0_b/chr2.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-2929ya7h/chr5.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-hgkfe1dq/chr10.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-mydkxlm7/chr22.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-j2wdw11i/chr17.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-z_dajg1d/chr15.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-na95q6a6/chr12.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-uuexp9ii/chr4.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-0uooacy1/chr7.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-vfcfnfbo/chr9.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-e8xvdfut/file-0_ke5xok/chr16.vcf to output location +[2020-11-27T15:55:07-0800] [MainThread] [I] [toil.leader] Job ended: 'unmerge' kind-unmerge/instance-e8xvdfut +[2020-11-27T15:55:07-0800] [MainThread] [I] [toil.leader] Job ended: 'unmerge' kind-unmerge/instance-n0yye4h5 +[2020-11-27T15:55:07-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-koznydha with job batch system ID: 291 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-0u_wnm58/chr13.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-c2bj25kq/chr12.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-nfmfvx5l/chr22.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-votlhp32/chr18.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-fft_cy45/chr17.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-xqzsu9s0/chr7.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-jo4ftcs4/chr9.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-b5ar7gc3/chrX.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-rwmqt2j3/chr19.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-lb85t2a5/chr8.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-x89p0q51/chr3.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-xrtj0_sq/chr16.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-p1_wdf3n/chr20.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-vz136l6i/chr21.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-59ttyo0x/chrY.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-tcg91l86/chr6.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-2r3q0g8u/chr2.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-hs3lzgs1/chr14.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-z0lunw99/chrM.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-d3z7omyq/chr4.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-foduo8hh/chr5.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-q_6cv926/chr10.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-2ob9ahri/chr15.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-eljm2b3r/chr1.vcf to output location +[2020-11-27T15:55:07-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:07: Exporting files/for-job/kind-unmerge/instance-n0yye4h5/file-f6ihpl78/chr11.vcf to output location +[2020-11-27T15:55:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpnw4bqxsf/worker_log.txt +[2020-11-27T15:55:08-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-koznydha +[2020-11-27T15:55:08-0800] [MainThread] [I] [toil.leader] Issued job 'wrap_unmerge' kind-EncapsulatedJob/instance-gi0_24pa with job batch system ID: 292 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T15:55:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpav_4w7wi/worker_log.txt +[2020-11-27T15:55:09-0800] [MainThread] [I] [toil.leader] Job ended: 'wrap_unmerge' kind-EncapsulatedJob/instance-gi0_24pa +[2020-11-27T15:55:09-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-fx0_pxis with job batch system ID: 293 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:55:10-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpzk82d3b5/worker_log.txt +[2020-11-27T15:55:10-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-fx0_pxis +[2020-11-27T15:55:10-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-rb7logwp with job batch system ID: 294 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:55:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpkyi6no1b/worker_log.txt +[2020-11-27T15:55:11-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-rb7logwp +[2020-11-27T15:55:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:55: Filtered SomaticSniper for TEST successfully +[2020-11-27T15:55:56-0800] [MainThread] [I] [toil.leader] Job ended: 'filter_somaticsniper' kind-filter_somaticsniper/instance-6mjhihuo +[2020-11-27T15:55:56-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-6llx2nmp with job batch system ID: 295 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:55:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpumpzu4cw/worker_log.txt +[2020-11-27T15:55:57-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-6llx2nmp +[2020-11-27T15:55:57-0800] [MainThread] [I] [toil.leader] Issued job 'unmerge' kind-unmerge/instance-l3y6r0t9 with job batch system ID: 296 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T15:55:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpwd9jmu9v/worker_log.txt +[2020-11-27T15:55:59-0800] [MainThread] [I] [toil.leader] Job ended: 'unmerge' kind-unmerge/instance-l3y6r0t9 +[2020-11-27T15:55:59-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-6llx2nmp with job batch system ID: 297 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-0gpgx9qz/chr4.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-pwgrmlv4/chr6.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-dt8fi506/chr7.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-_so2ows4/chr8.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-51wbiwfq/chr15.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-m30ubatz/chr16.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-n9_j861v/chrM.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-19x7mkdi/chr19.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-si74e3in/chr12.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-q3v44k0j/chrY.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-b__5dta5/chr18.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-pa68f_4b/chr21.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-zatrqi5p/chr2.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-nodgeiwi/chr17.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-w1o0agus/chr22.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-p63h052y/chr3.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-bb0uk9b6/chr10.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-0zy4r614/chr11.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-drg9d0yb/chr9.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-kh_stgbk/chrX.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-lxrs7d8a/chr1.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-33ve6v2d/chr14.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-th0si83o/chr13.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-5_sb331a/chr5.vcf to output location +[2020-11-27T15:55:59-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:55:59: Exporting files/for-job/kind-unmerge/instance-l3y6r0t9/file-lom1lrsv/chr20.vcf to output location +[2020-11-27T15:56:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmphe5of9tb/worker_log.txt +[2020-11-27T15:56:00-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-6llx2nmp +[2020-11-27T15:56:00-0800] [MainThread] [I] [toil.leader] Issued job 'filter_somaticsniper' kind-EncapsulatedJob/instance-u3pcg5lk with job batch system ID: 298 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:56:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_rfhtls4/worker_log.txt +[2020-11-27T15:56:01-0800] [MainThread] [I] [toil.leader] Job ended: 'filter_somaticsniper' kind-EncapsulatedJob/instance-u3pcg5lk +[2020-11-27T15:56:01-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-p6aqi182 with job batch system ID: 299 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:56:01-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-q74ncvyg with job batch system ID: 300 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:56:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpcl35airr/worker_log.txt +[2020-11-27T15:56:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpqbm07sdt/worker_log.txt +[2020-11-27T15:56:01-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-q74ncvyg +[2020-11-27T15:56:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_somaticsniper_full' kind-EncapsulatedJob/instance-sydh7m88 with job batch system ID: 301 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:56:01-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-p6aqi182 +[2020-11-27T15:56:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_pileup' kind-EncapsulatedJob/instance-xmq5os92 with job batch system ID: 302 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp27bqn3yt/worker_log.txt +[2020-11-27T15:56:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_somaticsniper_full' kind-EncapsulatedJob/instance-sydh7m88 +[2020-11-27T15:56:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1f869cw5/worker_log.txt +[2020-11-27T15:56:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_pileup' kind-EncapsulatedJob/instance-xmq5os92 +[2020-11-27T15:56:02-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-na5nlnmq with job batch system ID: 303 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:56:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpuv0h7cpm/worker_log.txt +[2020-11-27T15:56:03-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-na5nlnmq +[2020-11-27T15:57:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:57:28: Exporting files/for-job/kind-run_star/instance-qh34obz5/file-tzdz2g94/rnaAligned.toTranscriptome.out.bam to output location +[2020-11-27T15:57:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:57:28: Exporting files/for-job/kind-run_star/instance-qh34obz5/file-nmx5g1xc/rnaChimeric.out.junction to output location +[2020-11-27T15:57:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:57:28: Ran STAR on TEST successfully +[2020-11-27T15:57:28-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:57:28: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-y_lo_bvx/cleanup/file-am78xbx8/stream used 112.45% (60.3 GB [64780701696B] used, 53.7 GB [57610702127B] requested) at the end of its run. +[2020-11-27T15:57:38-0800] [MainThread] [I] [toil.leader] Job ended: 'run_star' kind-run_star/instance-qh34obz5 +[2020-11-27T15:57:38-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-n014_f_1 with job batch system ID: 304 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:57:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpinrf4g3s/worker_log.txt +[2020-11-27T15:57:39-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-n014_f_1 +[2020-11-27T15:57:39-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-jwa27gxg with job batch system ID: 305 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T15:57:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmprbx9_3ju/worker_log.txt +[2020-11-27T15:58:15-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-jwa27gxg +[2020-11-27T15:58:15-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-n014_f_1 with job batch system ID: 306 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:58:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:58:15: Ran samtools-sort on TEST:rna successfully +[2020-11-27T15:58:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:58:15: Exporting files/for-job/kind-EncapsulatedJob/instance-jwa27gxg/file-xw6xunbt/rna_sorted.bam to output location +[2020-11-27T15:58:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:58:15: Exporting files/for-job/kind-EncapsulatedJob/instance-jwa27gxg/file-dset1n7f/rna_genome_sorted.bam.bai to output location +[2020-11-27T15:58:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:58:15: Ran samtools-index on TEST:rna successfully +[2020-11-27T15:58:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpd4qehnqy/worker_log.txt +[2020-11-27T15:58:16-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-n014_f_1 +[2020-11-27T15:58:16-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-19ln8xui with job batch system ID: 307 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T15:58:17-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpf40qldcq/worker_log.txt +[2020-11-27T15:58:17-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-19ln8xui +[2020-11-27T15:58:17-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-mgdq08hc with job batch system ID: 308 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T15:58:17-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-i6wy4k24 with job batch system ID: 309 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-27T15:58:17-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-y0q8img4 with job batch system ID: 310 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-27T15:58:17-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpk3wap3aj/worker_log.txt +[2020-11-27T15:58:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpobmug5_7/worker_log.txt +[2020-11-27T15:58:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpw5nbq841/worker_log.txt +[2020-11-27T15:58:18-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-mgdq08hc +[2020-11-27T15:58:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_fusion' kind-run_fusion/instance-83jxfemz with job batch system ID: 311 and cores: 80, disk: 53.4 G, and memory: 48.6 G +[2020-11-27T15:58:18-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-i6wy4k24 +[2020-11-27T15:58:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_rsem' kind-run_rsem/instance-4q7mdxdg with job batch system ID: 312 and cores: 80, disk: 5.0 G, and memory: 2.0 G +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-y0q8img4 +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-zc4ncec4 with job batch system ID: 313 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-tqnu8l56 with job batch system ID: 314 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-hht82wzu with job batch system ID: 315 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-097yf3t5 with job batch system ID: 316 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-5iaf3waa with job batch system ID: 317 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-36ghujog with job batch system ID: 318 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-hvhruste with job batch system ID: 319 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-h7gmbwkp with job batch system ID: 320 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-o3wqly5f with job batch system ID: 321 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-tq73t52b with job batch system ID: 322 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-1ebrjhj8 with job batch system ID: 323 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-jgz9mu9a with job batch system ID: 324 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-qp73arin with job batch system ID: 325 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-3d5smzm7 with job batch system ID: 326 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-ow28qiwu with job batch system ID: 327 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-atufsa6u with job batch system ID: 328 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-88tfkfgg with job batch system ID: 329 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-y002g4r9 with job batch system ID: 330 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-0o7tl58g with job batch system ID: 331 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-h9fuyyh2 with job batch system ID: 332 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-u7f0udus with job batch system ID: 333 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-tvzoe0p1 with job batch system ID: 334 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-j9989bly with job batch system ID: 335 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-lo5qoutv with job batch system ID: 336 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-z9l9o61x with job batch system ID: 337 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T15:58:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpzxbehkce/worker_log.txt +[2020-11-27T15:58:19-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 15:58:19: Ran spawn_radia on TEST successfully +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpa9j5w982/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpnu5i9xpp/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpg2jorzyu/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1cqb4492/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpmz40kutd/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpiphg0mgy/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp88nesz0h/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp0ckwvwtu/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp2kd0_6cq/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdbg_e4m8/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpazn5ykvx/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmppzjnt5gi/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpcopr85ze/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgm26g3h_/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpf7e7e40b/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp32fl8cwz/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpwn9glxu0/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmps3a8h7vg/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpcbhyxbmk/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp2jpbol8a/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpc7tmhx14/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp65gxzg5r/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmphapxv8ar/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpw8kdqx4j/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpl0kzjgho/worker_log.txt +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-hvhruste +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-883erw0y with job batch system ID: 338 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-h9fuyyh2 +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-vio3oi8t with job batch system ID: 339 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-tqnu8l56 +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-cax7nt5c with job batch system ID: 340 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-zc4ncec4 +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-h2czw2cf with job batch system ID: 341 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-hht82wzu +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-8wsr28eg with job batch system ID: 342 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-097yf3t5 +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-tzskrbrh with job batch system ID: 343 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-5iaf3waa +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-vm3n1_ax with job batch system ID: 344 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-36ghujog +[2020-11-27T15:58:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-h5aeqrpp with job batch system ID: 345 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-o3wqly5f +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-4bo0z6ea with job batch system ID: 346 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-1ebrjhj8 +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-g0qywg9y with job batch system ID: 347 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-jgz9mu9a +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-u3r3g21y with job batch system ID: 348 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-qp73arin +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-_i22pcpj with job batch system ID: 349 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-tq73t52b +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-w544c6yq with job batch system ID: 350 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-h7gmbwkp +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-lpe5s5c1 with job batch system ID: 351 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-3d5smzm7 +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-kmevwjkz with job batch system ID: 352 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-ow28qiwu +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-eiv6p76r with job batch system ID: 353 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-atufsa6u +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-7r2q6grh with job batch system ID: 354 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-88tfkfgg +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-ns09m8c0 with job batch system ID: 355 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-y002g4r9 +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-jo48sx1l with job batch system ID: 356 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-0o7tl58g +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-wu4im_si with job batch system ID: 357 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-u7f0udus +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-t47j1mta with job batch system ID: 358 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-tvzoe0p1 +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-co5ii36y with job batch system ID: 359 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-j9989bly +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-5fureoef with job batch system ID: 360 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-lo5qoutv +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-766uu9_p with job batch system ID: 361 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-z9l9o61x +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-run_radia_perchrom/instance-_emujofj with job batch system ID: 362 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T15:58:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmps4k03j_7/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp80qifil1/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpzmtcqs_g/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7mj1fnbq/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpjgxb62fj/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp05o4gf1w/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpz4_qqnem/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpcwhjmzbl/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpvh5mr60t/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpnzxk5ydr/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpc1aluzju/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpyeuxxlwr/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpuompsfo7/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp91romtiq/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpfi_jxy0v/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmprat7gkfa/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpe0nghsz4/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbtcaxwo1/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpy5daob6y/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpn71ckhla/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmppmjo7qap/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpsklh6s9d/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp4slr0nx6/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp57gey0d8/worker_log.txt +[2020-11-27T15:58:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp93e4_0pn/worker_log.txt +[2020-11-27T16:01:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:01:56: Ran radia on TEST:chrM successfully +[2020-11-27T16:01:57-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-_emujofj +[2020-11-27T16:01:57-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-9m13dq1n with job batch system ID: 363 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:01:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpqojjzvxk/worker_log.txt +[2020-11-27T16:01:58-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-9m13dq1n +[2020-11-27T16:01:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-wwc8ravf with job batch system ID: 364 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:01:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpxd1wfppv/worker_log.txt +[2020-11-27T16:03:48-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:03:48: Ran radia on TEST:chrY successfully +[2020-11-27T16:03:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-766uu9_p +[2020-11-27T16:03:48-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-3xkmaqmz with job batch system ID: 365 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:03:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpojl2i6vp/worker_log.txt +[2020-11-27T16:03:49-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-3xkmaqmz +[2020-11-27T16:03:49-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-souyyd8g with job batch system ID: 366 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:03:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdskfhe3j/worker_log.txt +[2020-11-27T16:05:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:05:01: Exporting files/for-job/kind-run_filter_radia/instance-souyyd8g/file-krkmee4s/chrY.vcf to output location +[2020-11-27T16:05:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:05:01: Ran filter-radia on TEST:chrY successfully +[2020-11-27T16:05:01-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:05:01: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-3xkmaqmz/cleanup/file-axbygu0c/stream used 123.23% (6.5 GB [7020257280B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:05:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-souyyd8g +[2020-11-27T16:05:02-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-vi08tsi_ with job batch system ID: 367 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:05:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpyxuob3oi/worker_log.txt +[2020-11-27T16:05:03-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-vi08tsi_ +[2020-11-27T16:05:03-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-3xkmaqmz with job batch system ID: 368 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:05:04-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp6znbcn_w/worker_log.txt +[2020-11-27T16:05:04-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-3xkmaqmz +[2020-11-27T16:05:04-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-lo5qoutv with job batch system ID: 369 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:05:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpe5sc8bs4/worker_log.txt +[2020-11-27T16:05:05-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-lo5qoutv +[2020-11-27T16:05:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:05:28: Exporting files/for-job/kind-run_filter_radia/instance-wwc8ravf/file-bxnv2now/chrM.vcf to output location +[2020-11-27T16:05:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:05:28: Ran filter-radia on TEST:chrM successfully +[2020-11-27T16:05:28-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:05:28: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-9m13dq1n/cleanup/file-wikf9iuk/stream used 123.22% (6.5 GB [7019675648B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:05:29-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-wwc8ravf +[2020-11-27T16:05:29-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-xjgvs9cr with job batch system ID: 370 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:05:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpm1e_tv2p/worker_log.txt +[2020-11-27T16:05:30-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-xjgvs9cr +[2020-11-27T16:05:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-9m13dq1n with job batch system ID: 371 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:05:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp4y9ai000/worker_log.txt +[2020-11-27T16:05:30-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-9m13dq1n +[2020-11-27T16:05:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-z9l9o61x with job batch system ID: 372 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:05:31-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpzrq9mm28/worker_log.txt +[2020-11-27T16:05:31-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-z9l9o61x +[2020-11-27T16:09:06-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:09:06: Ran radia on TEST:chr21 successfully +[2020-11-27T16:09:07-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-t47j1mta +[2020-11-27T16:09:07-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-i4q6no1g with job batch system ID: 373 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:09:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpvbxfhhrd/worker_log.txt +[2020-11-27T16:09:08-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-i4q6no1g +[2020-11-27T16:09:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-6wfb4cko with job batch system ID: 374 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:09:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7kdg4jfx/worker_log.txt +[2020-11-27T16:10:24-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:10:24: Exporting files/for-job/kind-run_filter_radia/instance-6wfb4cko/file-4g7usyiz/chr21.vcf to output location +[2020-11-27T16:10:24-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:10:24: Ran filter-radia on TEST:chr21 successfully +[2020-11-27T16:10:24-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:10:24: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-i4q6no1g/cleanup/file-o17pqbyp/stream used 123.18% (6.5 GB [7017537536B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:10:25-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-6wfb4cko +[2020-11-27T16:10:25-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-e4s_7ilo with job batch system ID: 375 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:10:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp0hwedcoe/worker_log.txt +[2020-11-27T16:10:26-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-e4s_7ilo +[2020-11-27T16:10:26-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-i4q6no1g with job batch system ID: 376 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:10:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1rpsxl7a/worker_log.txt +[2020-11-27T16:10:27-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-i4q6no1g +[2020-11-27T16:10:27-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-u7f0udus with job batch system ID: 377 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:10:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpz2wkiinr/worker_log.txt +[2020-11-27T16:10:27-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-u7f0udus +[2020-11-27T16:11:39-0800] [MainThread] [I] [toil.leader] 48 jobs are running, 1 jobs are issued and waiting to run +[2020-11-27T16:11:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:11:43: Ran radia on TEST:chr22 successfully +[2020-11-27T16:11:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-co5ii36y +[2020-11-27T16:11:44-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-om91uztx with job batch system ID: 378 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:11:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpmh7w0yky/worker_log.txt +[2020-11-27T16:11:45-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-om91uztx +[2020-11-27T16:11:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-d8nzg5y8 with job batch system ID: 379 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:11:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpjbsk0452/worker_log.txt +[2020-11-27T16:13:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:13:03: Exporting files/for-job/kind-run_filter_radia/instance-d8nzg5y8/file-fwga3qg7/chr22.vcf to output location +[2020-11-27T16:13:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:13:03: Ran filter-radia on TEST:chr22 successfully +[2020-11-27T16:13:03-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:13:03: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-om91uztx/cleanup/file-x8djgsu8/stream used 123.19% (6.5 GB [7018123264B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:13:04-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-d8nzg5y8 +[2020-11-27T16:13:04-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-n7fh9fru with job batch system ID: 380 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:13:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpw8j9yqk5/worker_log.txt +[2020-11-27T16:13:05-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-n7fh9fru +[2020-11-27T16:13:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-om91uztx with job batch system ID: 381 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:13:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpfnkmoo47/worker_log.txt +[2020-11-27T16:13:06-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-om91uztx +[2020-11-27T16:13:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-tvzoe0p1 with job batch system ID: 382 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:13:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp71cbgqku/worker_log.txt +[2020-11-27T16:13:07-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-tvzoe0p1 +[2020-11-27T16:14:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:14:20: Ran radia on TEST:chr19 successfully +[2020-11-27T16:14:20-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-wu4im_si +[2020-11-27T16:14:20-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-mjsmnbj0 with job batch system ID: 383 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:14:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpfb22xh1j/worker_log.txt +[2020-11-27T16:14:22-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-mjsmnbj0 +[2020-11-27T16:14:22-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-s67wbk4l with job batch system ID: 384 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:14:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp2rsp4x5q/worker_log.txt +[2020-11-27T16:15:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:15:39: Ran radia on TEST:chr20 successfully +[2020-11-27T16:15:39-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-vio3oi8t +[2020-11-27T16:15:39-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-u1rj80l2 with job batch system ID: 385 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:15:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp6ywb1arp/worker_log.txt +[2020-11-27T16:15:41-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-u1rj80l2 +[2020-11-27T16:15:41-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-ugi5vfkp with job batch system ID: 386 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:15:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmps1t4t4ha/worker_log.txt +[2020-11-27T16:15:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:15:50: Exporting files/for-job/kind-run_filter_radia/instance-s67wbk4l/file-eaiy_2wl/chr19.vcf to output location +[2020-11-27T16:15:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:15:50: Ran filter-radia on TEST:chr19 successfully +[2020-11-27T16:15:50-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:15:50: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-mjsmnbj0/cleanup/file-0ysnpg1x/stream used 123.21% (6.5 GB [7019335680B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:15:51-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-s67wbk4l +[2020-11-27T16:15:51-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-jlz_0mq_ with job batch system ID: 387 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:15:52-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpogbunb4l/worker_log.txt +[2020-11-27T16:15:52-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-jlz_0mq_ +[2020-11-27T16:15:52-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-mjsmnbj0 with job batch system ID: 388 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:15:52-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpj3w7fd2j/worker_log.txt +[2020-11-27T16:15:52-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-mjsmnbj0 +[2020-11-27T16:15:52-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-0o7tl58g with job batch system ID: 389 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:15:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpephd3m9j/worker_log.txt +[2020-11-27T16:15:53-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-0o7tl58g +[2020-11-27T16:17:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:17:00: Exporting files/for-job/kind-run_filter_radia/instance-ugi5vfkp/file-ku7630hl/chr20.vcf to output location +[2020-11-27T16:17:00-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:17:00: Ran filter-radia on TEST:chr20 successfully +[2020-11-27T16:17:00-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:17:00: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-u1rj80l2/cleanup/file-yzz3r09e/stream used 123.20% (6.5 GB [7018471424B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:17:01-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-ugi5vfkp +[2020-11-27T16:17:01-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-54qpanz3 with job batch system ID: 390 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:17:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpx4gp5550/worker_log.txt +[2020-11-27T16:17:02-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-54qpanz3 +[2020-11-27T16:17:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-u1rj80l2 with job batch system ID: 391 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:17:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpw3k4qcan/worker_log.txt +[2020-11-27T16:17:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-u1rj80l2 +[2020-11-27T16:17:03-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-h9fuyyh2 with job batch system ID: 392 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:17:04-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpmyx8qdhv/worker_log.txt +[2020-11-27T16:17:04-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-h9fuyyh2 +[2020-11-27T16:17:25-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:17:25: Ran radia on TEST:chr18 successfully +[2020-11-27T16:17:25-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-jo48sx1l +[2020-11-27T16:17:25-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-qyn3pw8p with job batch system ID: 393 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:17:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpb0_oqzux/worker_log.txt +[2020-11-27T16:17:26-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-qyn3pw8p +[2020-11-27T16:17:26-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-kjkqkzfv with job batch system ID: 394 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:17:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp8eye_q3z/worker_log.txt +[2020-11-27T16:18:47-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:18:47: Exporting files/for-job/kind-run_filter_radia/instance-kjkqkzfv/file-fucq4am9/chr18.vcf to output location +[2020-11-27T16:18:47-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:18:47: Ran filter-radia on TEST:chr18 successfully +[2020-11-27T16:18:47-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:18:47: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-qyn3pw8p/cleanup/file-rtqh09z3/stream used 123.21% (6.5 GB [7019204608B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:18:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-kjkqkzfv +[2020-11-27T16:18:48-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-9khkdkjc with job batch system ID: 395 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:18:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp9s5y9har/worker_log.txt +[2020-11-27T16:18:49-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-9khkdkjc +[2020-11-27T16:18:49-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-qyn3pw8p with job batch system ID: 396 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:18:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpa_efwhe5/worker_log.txt +[2020-11-27T16:18:50-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-qyn3pw8p +[2020-11-27T16:18:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-y002g4r9 with job batch system ID: 397 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:18:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_pm3zdql/worker_log.txt +[2020-11-27T16:18:51-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-y002g4r9 +[2020-11-27T16:19:12-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:19:12: Ran radia on TEST:chr16 successfully +[2020-11-27T16:19:13-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-7r2q6grh +[2020-11-27T16:19:13-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-mlb8mlfl with job batch system ID: 398 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:19:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpmf64943w/worker_log.txt +[2020-11-27T16:19:14-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-mlb8mlfl +[2020-11-27T16:19:14-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-03_wuvbl with job batch system ID: 399 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:19:15-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpw8xivd54/worker_log.txt +[2020-11-27T16:20:04-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:20:04: Ran radia on TEST:chr17 successfully +[2020-11-27T16:20:04-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-ns09m8c0 +[2020-11-27T16:20:04-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-hm1aire4 with job batch system ID: 400 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:20:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpss_hcih4/worker_log.txt +[2020-11-27T16:20:05-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-hm1aire4 +[2020-11-27T16:20:05-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-wsev1gg0 with job batch system ID: 401 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:20:06-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:20:06: Ran radia on TEST:chr13 successfully +[2020-11-27T16:20:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmptns2arxa/worker_log.txt +[2020-11-27T16:20:06-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-_i22pcpj +[2020-11-27T16:20:06-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-g4mmbys3 with job batch system ID: 402 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:20:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp2ux4buyi/worker_log.txt +[2020-11-27T16:20:07-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-g4mmbys3 +[2020-11-27T16:20:07-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-15ft3e8_ with job batch system ID: 403 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:20:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmptywl3tzm/worker_log.txt +[2020-11-27T16:20:24-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:20:24: Ran radia on TEST:chr15 successfully +[2020-11-27T16:20:25-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-eiv6p76r +[2020-11-27T16:20:25-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-erb6o762 with job batch system ID: 404 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:20:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbq56jovz/worker_log.txt +[2020-11-27T16:20:26-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-erb6o762 +[2020-11-27T16:20:26-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-jwc7csrp with job batch system ID: 405 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:20:27-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpfe7xl0ov/worker_log.txt +[2020-11-27T16:20:41-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:20:41: Ran radia on TEST:chrX successfully +[2020-11-27T16:20:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-5fureoef +[2020-11-27T16:20:41-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-_5ut7fs1 with job batch system ID: 406 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:20:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbzmaarwe/worker_log.txt +[2020-11-27T16:20:43-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-_5ut7fs1 +[2020-11-27T16:20:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-s0kh1v45 with job batch system ID: 407 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:20:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpfclqwb9h/worker_log.txt +[2020-11-27T16:20:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:20:46: Exporting files/for-job/kind-run_filter_radia/instance-03_wuvbl/file-icl5ngjh/chr16.vcf to output location +[2020-11-27T16:20:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:20:46: Ran filter-radia on TEST:chr16 successfully +[2020-11-27T16:20:46-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:20:46: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-mlb8mlfl/cleanup/file-zgd5mkvw/stream used 123.24% (6.5 GB [7020969984B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:20:47-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-03_wuvbl +[2020-11-27T16:20:47-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-uq1l87ot with job batch system ID: 408 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:20:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp2owm9jm4/worker_log.txt +[2020-11-27T16:20:48-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-uq1l87ot +[2020-11-27T16:20:48-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-mlb8mlfl with job batch system ID: 409 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:20:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpwl6szfc3/worker_log.txt +[2020-11-27T16:20:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-mlb8mlfl +[2020-11-27T16:20:48-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-atufsa6u with job batch system ID: 410 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:20:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpurc90_sy/worker_log.txt +[2020-11-27T16:20:49-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-atufsa6u +[2020-11-27T16:21:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:21:35: Exporting files/for-job/kind-run_filter_radia/instance-15ft3e8_/file-m0yjzjlz/chr13.vcf to output location +[2020-11-27T16:21:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:21:35: Ran filter-radia on TEST:chr13 successfully +[2020-11-27T16:21:35-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:21:35: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-g4mmbys3/cleanup/file-da5hluo3/stream used 123.19% (6.5 GB [7017963520B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:21:36-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-15ft3e8_ +[2020-11-27T16:21:36-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-n0o5u2jj with job batch system ID: 411 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:21:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpl864j58o/worker_log.txt +[2020-11-27T16:21:37-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-n0o5u2jj +[2020-11-27T16:21:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-g4mmbys3 with job batch system ID: 412 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:21:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmppasbplyl/worker_log.txt +[2020-11-27T16:21:38-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-g4mmbys3 +[2020-11-27T16:21:38-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-qp73arin with job batch system ID: 413 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:21:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp89ae_wih/worker_log.txt +[2020-11-27T16:21:39-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-qp73arin +[2020-11-27T16:21:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:21:50: Exporting files/for-job/kind-run_filter_radia/instance-wsev1gg0/file-59_a3ggh/chr17.vcf to output location +[2020-11-27T16:21:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:21:50: Ran filter-radia on TEST:chr17 successfully +[2020-11-27T16:21:50-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:21:50: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-hm1aire4/cleanup/file-8bgrw7w8/stream used 123.23% (6.5 GB [7020040192B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:21:51-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-wsev1gg0 +[2020-11-27T16:21:51-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-iq847sr0 with job batch system ID: 414 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:21:52-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp23cae9pb/worker_log.txt +[2020-11-27T16:21:52-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-iq847sr0 +[2020-11-27T16:21:52-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-hm1aire4 with job batch system ID: 415 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:21:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpqk6ojucg/worker_log.txt +[2020-11-27T16:21:53-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-hm1aire4 +[2020-11-27T16:21:53-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-88tfkfgg with job batch system ID: 416 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:21:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpg63mxj8z/worker_log.txt +[2020-11-27T16:21:54-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-88tfkfgg +[2020-11-27T16:21:58-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:21:58: Exporting files/for-job/kind-run_filter_radia/instance-jwc7csrp/file-fzoec8so/chr15.vcf to output location +[2020-11-27T16:21:58-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:21:58: Ran filter-radia on TEST:chr15 successfully +[2020-11-27T16:21:58-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:21:58: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-erb6o762/cleanup/file-pxqci8q8/stream used 123.19% (6.5 GB [7018205184B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:21:59-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-jwc7csrp +[2020-11-27T16:21:59-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-p2qcub0u with job batch system ID: 417 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:21:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpck4d___4/worker_log.txt +[2020-11-27T16:22:00-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-p2qcub0u +[2020-11-27T16:22:00-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-erb6o762 with job batch system ID: 418 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:22:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpp6rrvste/worker_log.txt +[2020-11-27T16:22:01-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-erb6o762 +[2020-11-27T16:22:01-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-ow28qiwu with job batch system ID: 419 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:22:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp61tadejy/worker_log.txt +[2020-11-27T16:22:01-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-ow28qiwu +[2020-11-27T16:22:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:22:09: Exporting files/for-job/kind-run_filter_radia/instance-s0kh1v45/file-eox5jiwe/chrX.vcf to output location +[2020-11-27T16:22:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:22:09: Ran filter-radia on TEST:chrX successfully +[2020-11-27T16:22:09-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:22:09: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-_5ut7fs1/cleanup/file-1p32a6ec/stream used 123.20% (6.5 GB [7018270720B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:22:10-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-s0kh1v45 +[2020-11-27T16:22:10-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ara1qhch with job batch system ID: 420 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:22:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp6sh5rtnc/worker_log.txt +[2020-11-27T16:22:11-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ara1qhch +[2020-11-27T16:22:11-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-_5ut7fs1 with job batch system ID: 421 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:22:12-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpoclg3t2j/worker_log.txt +[2020-11-27T16:22:12-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-_5ut7fs1 +[2020-11-27T16:22:12-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-j9989bly with job batch system ID: 422 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:22:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpri44wnbn/worker_log.txt +[2020-11-27T16:22:13-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-j9989bly +[2020-11-27T16:22:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:22:40: Ran radia on TEST:chr14 successfully +[2020-11-27T16:22:40-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-kmevwjkz +[2020-11-27T16:22:40-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-7heyypsn with job batch system ID: 423 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:22:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpb_0k692h/worker_log.txt +[2020-11-27T16:22:41-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-7heyypsn +[2020-11-27T16:22:41-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-priap7ai with job batch system ID: 424 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:22:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpkec2zgr8/worker_log.txt +[2020-11-27T16:23:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:23:57: Exporting files/for-job/kind-run_fusion/instance-83jxfemz/file-txh6wu7z/star-fusion.fusion_candidates.final.abridged to output location +[2020-11-27T16:23:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:23:57: Exporting files/for-job/kind-run_fusion/instance-83jxfemz/file-eghjvmb2/FusionInspector.fusion_predictions.final.abridged.FFPM to output location +[2020-11-27T16:23:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:23:57: Exporting files/for-job/kind-run_fusion/instance-83jxfemz/file-zj0byl6i/FusionInspector.gmap_trinity_GG.fusions.fasta to output location +[2020-11-27T16:23:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:23:57: Exporting files/for-job/kind-run_fusion/instance-83jxfemz/file-f29cl34x/FusionInspector.gmap_trinity_GG.fusions.gff3 to output location +[2020-11-27T16:23:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:23:57: Exporting files/for-job/kind-run_fusion/instance-83jxfemz/file-rikvp7rj/fusion.final to output location +[2020-11-27T16:23:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:23:57: Ran STAR-Fusion on TEST successfully +[2020-11-27T16:23:57-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:23:57: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-mgdq08hc/cleanup/file-b3pef99a/stream used 111.61% (59.6 GB [64029347840B] used, 53.4 GB [57369019908B] requested) at the end of its run. +[2020-11-27T16:23:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:23:57: Exporting files/for-job/kind-run_fusion/instance-83jxfemz/file-dcqm14ul/fusion_results.bedpe to output location +[2020-11-27T16:23:57-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:23:57: Reformatted STAR-Fusion output for TEST successfully +[2020-11-27T16:24:05-0800] [MainThread] [I] [toil.leader] Job ended: 'run_fusion' kind-run_fusion/instance-83jxfemz +[2020-11-27T16:24:05-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-b1y0y5c9 with job batch system ID: 425 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:24:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmplh4kqnuz/worker_log.txt +[2020-11-27T16:24:05-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp6ixmzb_v/worker_log.txt +[2020-11-27T16:24:06-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-b1y0y5c9 +[2020-11-27T16:24:06-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-7wszw7s6 with job batch system ID: 426 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:24:06-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpq6e8y7pf/worker_log.txt +[2020-11-27T16:24:06-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-7wszw7s6 +[2020-11-27T16:24:06-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-p3i6d1f0 with job batch system ID: 427 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:24:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpjk_vn8_y/worker_log.txt +[2020-11-27T16:24:07-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-p3i6d1f0 +[2020-11-27T16:24:07-0800] [MainThread] [I] [toil.leader] Issued job 'delete_fastqs' kind-delete_fastqs/instance-xrqkr_wb with job batch system ID: 428 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T16:24:07-0800] [MainThread] [I] [toil.leader] Issued job 'delete_fastqs' kind-delete_fastqs/instance-xdvsn8pw with job batch system ID: 429 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T16:24:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp9mjev8aq/worker_log.txt +[2020-11-27T16:24:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbz89t2gi/worker_log.txt +[2020-11-27T16:24:08-0800] [MainThread] [I] [toil.leader] Job ended: 'delete_fastqs' kind-delete_fastqs/instance-xdvsn8pw +[2020-11-27T16:24:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:24:09: Deleting "TEST:normal_dna_fastq_2" from the filestore. +[2020-11-27T16:24:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:24:09: Deleting "TEST:tumor_dna_fastq_2" from the filestore. +[2020-11-27T16:24:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:24:09: Deleting "TEST:tumor_rna_fastq_1" from the filestore. +[2020-11-27T16:24:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:24:09: Deleting "TEST:tumor_dna_fastq_1" from the filestore. +[2020-11-27T16:24:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:24:09: Deleting "TEST:normal_dna_fastq_1" from the filestore. +[2020-11-27T16:24:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:24:09: Deleting "TEST:tumor_rna_fastq_2" from the filestore. +[2020-11-27T16:24:09-0800] [MainThread] [I] [toil.leader] Job ended: 'delete_fastqs' kind-delete_fastqs/instance-xrqkr_wb +[2020-11-27T16:24:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:24:09: Exporting files/for-job/kind-run_filter_radia/instance-priap7ai/file-070u7dq7/chr14.vcf to output location +[2020-11-27T16:24:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:24:09: Ran filter-radia on TEST:chr14 successfully +[2020-11-27T16:24:09-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:24:09: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-7heyypsn/cleanup/file-x49imj44/stream used 123.18% (6.5 GB [7017529344B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:24:10-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-priap7ai +[2020-11-27T16:24:10-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-m6b31ibl with job batch system ID: 430 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:24:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp3glpo444/worker_log.txt +[2020-11-27T16:24:11-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-m6b31ibl +[2020-11-27T16:24:11-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-7heyypsn with job batch system ID: 431 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:24:12-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp4w623lq3/worker_log.txt +[2020-11-27T16:24:12-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-7heyypsn +[2020-11-27T16:24:12-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-3d5smzm7 with job batch system ID: 432 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:24:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpj9inv7xa/worker_log.txt +[2020-11-27T16:24:13-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-3d5smzm7 +[2020-11-27T16:26:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:26:09: Ran radia on TEST:chr9 successfully +[2020-11-27T16:26:10-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-4bo0z6ea +[2020-11-27T16:26:10-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-w9fmv75_ with job batch system ID: 433 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:26:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpi1q6gl20/worker_log.txt +[2020-11-27T16:26:11-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-w9fmv75_ +[2020-11-27T16:26:11-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-9l01qvil with job batch system ID: 434 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:26:12-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpkp71fwsr/worker_log.txt +[2020-11-27T16:27:13-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:27:13: Ran radia on TEST:chr11 successfully +[2020-11-27T16:27:14-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-g0qywg9y +[2020-11-27T16:27:14-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-lcwnjgp_ with job batch system ID: 435 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:27:15-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp5371efbm/worker_log.txt +[2020-11-27T16:27:15-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-lcwnjgp_ +[2020-11-27T16:27:15-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-48ww6n13 with job batch system ID: 436 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:27:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp4unyfbha/worker_log.txt +[2020-11-27T16:27:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:27:40: Exporting files/for-job/kind-run_filter_radia/instance-9l01qvil/file-nb47boqv/chr9.vcf to output location +[2020-11-27T16:27:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:27:40: Ran filter-radia on TEST:chr9 successfully +[2020-11-27T16:27:40-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:27:40: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-w9fmv75_/cleanup/file-8n5e1kgr/stream used 123.19% (6.5 GB [7017963520B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:27:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-9l01qvil +[2020-11-27T16:27:41-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-cpqr_4ks with job batch system ID: 437 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:27:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp4h15rkw4/worker_log.txt +[2020-11-27T16:27:42-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-cpqr_4ks +[2020-11-27T16:27:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-w9fmv75_ with job batch system ID: 438 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:27:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:27:42: Ran radia on TEST:chr10 successfully +[2020-11-27T16:27:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp09w9vtwg/worker_log.txt +[2020-11-27T16:27:42-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-w9fmv75_ +[2020-11-27T16:27:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-o3wqly5f with job batch system ID: 439 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:27:43-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-w544c6yq +[2020-11-27T16:27:43-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-e2_9hoku with job batch system ID: 440 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:27:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp52t8s3jd/worker_log.txt +[2020-11-27T16:27:43-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-o3wqly5f +[2020-11-27T16:27:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpl8s9162k/worker_log.txt +[2020-11-27T16:27:44-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-e2_9hoku +[2020-11-27T16:27:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-qh5xv1vw with job batch system ID: 441 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:27:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7sqqcplt/worker_log.txt +[2020-11-27T16:28:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:28:09: Ran radia on TEST:chr12 successfully +[2020-11-27T16:28:09-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-u3r3g21y +[2020-11-27T16:28:09-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-uy_uhext with job batch system ID: 442 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:28:10-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpd0w03xjy/worker_log.txt +[2020-11-27T16:28:11-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-uy_uhext +[2020-11-27T16:28:11-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-343pjdbi with job batch system ID: 443 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:28:12-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmplul3s75i/worker_log.txt +[2020-11-27T16:28:51-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:28:51: Exporting files/for-job/kind-run_filter_radia/instance-48ww6n13/file-wglomwmm/chr11.vcf to output location +[2020-11-27T16:28:51-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:28:51: Ran filter-radia on TEST:chr11 successfully +[2020-11-27T16:28:51-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:28:51: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-lcwnjgp_/cleanup/file-fwgpjakn/stream used 123.22% (6.5 GB [7019872256B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:28:52-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:28:52: Exporting files/for-job/kind-run_rsem/instance-4q7mdxdg/file-40_i1p27/rsem.genes.results to output location +[2020-11-27T16:28:52-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:28:52: Exporting files/for-job/kind-run_rsem/instance-4q7mdxdg/file-8454zzd9/rsem.isoforms.results to output location +[2020-11-27T16:28:52-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:28:52: Ran rsem on TEST successfully +[2020-11-27T16:28:53-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-48ww6n13 +[2020-11-27T16:28:53-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-jhk4ul1o with job batch system ID: 444 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:28:53-0800] [MainThread] [I] [toil.leader] Job ended: 'run_rsem' kind-run_rsem/instance-4q7mdxdg +[2020-11-27T16:28:53-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-u6qsfqqy with job batch system ID: 445 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:28:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpk7vtuesn/worker_log.txt +[2020-11-27T16:28:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpqudhqbid/worker_log.txt +[2020-11-27T16:28:54-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-jhk4ul1o +[2020-11-27T16:28:54-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-lcwnjgp_ with job batch system ID: 446 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:28:54-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-u6qsfqqy +[2020-11-27T16:28:54-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-j3z8ggs3 with job batch system ID: 447 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:28:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbi4ssrxv/worker_log.txt +[2020-11-27T16:28:54-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-lcwnjgp_ +[2020-11-27T16:28:54-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-1ebrjhj8 with job batch system ID: 448 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:28:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpy0wnclhl/worker_log.txt +[2020-11-27T16:28:55-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-j3z8ggs3 +[2020-11-27T16:28:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_mhc_gene_assessment' kind-run_mhc_gene_assessment/instance-4ym33ava with job batch system ID: 449 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T16:28:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_itx_resistance_assessment' kind-run_itx_resistance_assessment/instance-os81mkai with job batch system ID: 450 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T16:28:55-0800] [MainThread] [I] [toil.leader] Issued job 'run_car_t_validity_assessment' kind-run_car_t_validity_assessment/instance-5ul1lt8s with job batch system ID: 451 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T16:28:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpt1lbjbqz/worker_log.txt +[2020-11-27T16:28:55-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-1ebrjhj8 +[2020-11-27T16:28:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp0f362f5i/worker_log.txt +[2020-11-27T16:28:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmphljjg_99/worker_log.txt +[2020-11-27T16:28:56-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgfe0z1eh/worker_log.txt +[2020-11-27T16:28:56-0800] [MainThread] [I] [toil.leader] Job ended: 'run_itx_resistance_assessment' kind-run_itx_resistance_assessment/instance-os81mkai +[2020-11-27T16:28:56-0800] [MainThread] [I] [toil.leader] Issued job 'assess_itx_resistance' kind-assess_itx_resistance/instance-1m0san8n with job batch system ID: 452 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T16:28:56-0800] [MainThread] [I] [toil.leader] Job ended: 'run_car_t_validity_assessment' kind-run_car_t_validity_assessment/instance-5ul1lt8s +[2020-11-27T16:28:56-0800] [MainThread] [I] [toil.leader] Issued job 'assess_car_t_validity' kind-assess_car_t_validity/instance-imwyxh9j with job batch system ID: 453 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T16:28:56-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mhc_gene_assessment' kind-run_mhc_gene_assessment/instance-4ym33ava +[2020-11-27T16:28:56-0800] [MainThread] [I] [toil.leader] Issued job 'assess_mhc_genes' kind-assess_mhc_genes/instance-ywoqikwz with job batch system ID: 454 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T16:28:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpe6r3u9fk/worker_log.txt +[2020-11-27T16:28:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpwmi1pl3f/worker_log.txt +[2020-11-27T16:28:57-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmputfcpgip/worker_log.txt +[2020-11-27T16:28:58-0800] [MainThread] [I] [toil.leader] Job ended: 'assess_car_t_validity' kind-assess_car_t_validity/instance-imwyxh9j +[2020-11-27T16:28:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_car_t_validity_assessment' kind-run_car_t_validity_assessment/instance-5ul1lt8s with job batch system ID: 455 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T16:28:58-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:28:58: Exporting files/for-job/kind-assess_itx_resistance/instance-1m0san8n/file-z7d45ioy/immunotherapy_resistance_report.txt to output location +[2020-11-27T16:28:58-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:28:58: Ran create immunotherapy resistance report on TEST successfully +[2020-11-27T16:28:58-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:28:58: Exporting files/for-job/kind-assess_car_t_validity/instance-imwyxh9j/file-y9d883bo/car_t_target_report.txt to output location +[2020-11-27T16:28:58-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:28:58: Ran car t validity assessment on TEST successfully +[2020-11-27T16:28:58-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:28:58: Exporting files/for-job/kind-assess_mhc_genes/instance-ywoqikwz/file-alfy6dp6/mhc_pathway_report.txt to output location +[2020-11-27T16:28:58-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:28:58: Ran mhc gene assessment on TEST successfully +[2020-11-27T16:28:58-0800] [MainThread] [I] [toil.leader] Job ended: 'assess_itx_resistance' kind-assess_itx_resistance/instance-1m0san8n +[2020-11-27T16:28:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_itx_resistance_assessment' kind-run_itx_resistance_assessment/instance-os81mkai with job batch system ID: 456 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T16:28:58-0800] [MainThread] [I] [toil.leader] Job ended: 'assess_mhc_genes' kind-assess_mhc_genes/instance-ywoqikwz +[2020-11-27T16:28:58-0800] [MainThread] [I] [toil.leader] Issued job 'run_mhc_gene_assessment' kind-run_mhc_gene_assessment/instance-4ym33ava with job batch system ID: 457 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T16:28:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpi8mbicba/worker_log.txt +[2020-11-27T16:28:59-0800] [MainThread] [I] [toil.leader] Job ended: 'run_car_t_validity_assessment' kind-run_car_t_validity_assessment/instance-5ul1lt8s +[2020-11-27T16:28:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpumwf67vt/worker_log.txt +[2020-11-27T16:28:59-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpn697losd/worker_log.txt +[2020-11-27T16:28:59-0800] [MainThread] [I] [toil.leader] Job ended: 'run_itx_resistance_assessment' kind-run_itx_resistance_assessment/instance-os81mkai +[2020-11-27T16:28:59-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mhc_gene_assessment' kind-run_mhc_gene_assessment/instance-4ym33ava +[2020-11-27T16:29:18-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:29:18: Exporting files/for-job/kind-run_filter_radia/instance-qh5xv1vw/file-ogmo_vrc/chr10.vcf to output location +[2020-11-27T16:29:18-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:29:18: Ran filter-radia on TEST:chr10 successfully +[2020-11-27T16:29:18-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:29:18: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-e2_9hoku/cleanup/file-d975mx01/stream used 123.22% (6.5 GB [7019659264B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:29:19-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-qh5xv1vw +[2020-11-27T16:29:19-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-_9u_srqw with job batch system ID: 458 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:29:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp3ojqcq_g/worker_log.txt +[2020-11-27T16:29:20-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-_9u_srqw +[2020-11-27T16:29:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-e2_9hoku with job batch system ID: 459 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:29:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpxam4grv7/worker_log.txt +[2020-11-27T16:29:20-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-e2_9hoku +[2020-11-27T16:29:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-tq73t52b with job batch system ID: 460 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:29:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp9a5g5znj/worker_log.txt +[2020-11-27T16:29:21-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-tq73t52b +[2020-11-27T16:29:52-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:29:52: Exporting files/for-job/kind-run_filter_radia/instance-343pjdbi/file-__12hxta/chr12.vcf to output location +[2020-11-27T16:29:52-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:29:52: Ran filter-radia on TEST:chr12 successfully +[2020-11-27T16:29:52-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:29:52: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-uy_uhext/cleanup/file-5xb15ert/stream used 123.21% (6.5 GB [7019069440B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:29:53-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-343pjdbi +[2020-11-27T16:29:53-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-t84joku6 with job batch system ID: 461 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:29:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_6n2yr7u/worker_log.txt +[2020-11-27T16:29:54-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-t84joku6 +[2020-11-27T16:29:54-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-uy_uhext with job batch system ID: 462 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:29:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp62z4fw0q/worker_log.txt +[2020-11-27T16:29:54-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-uy_uhext +[2020-11-27T16:29:54-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-jgz9mu9a with job batch system ID: 463 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:29:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpg0uamcsv/worker_log.txt +[2020-11-27T16:29:55-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-jgz9mu9a +[2020-11-27T16:31:29-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:31:29: Ran radia on TEST:chr8 successfully +[2020-11-27T16:31:30-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-lpe5s5c1 +[2020-11-27T16:31:30-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-p55975qr with job batch system ID: 464 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:31:31-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpkmovyopw/worker_log.txt +[2020-11-27T16:31:31-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-p55975qr +[2020-11-27T16:31:31-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-de4euzyv with job batch system ID: 465 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:31:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpzj_65kgl/worker_log.txt +[2020-11-27T16:33:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:33:09: Exporting files/for-job/kind-run_filter_radia/instance-de4euzyv/file-ih00_mjo/chr8.vcf to output location +[2020-11-27T16:33:09-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:33:09: Ran filter-radia on TEST:chr8 successfully +[2020-11-27T16:33:09-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:33:09: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-p55975qr/cleanup/file-0gprzxks/stream used 123.20% (6.5 GB [7018586112B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:33:10-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-de4euzyv +[2020-11-27T16:33:10-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-yb02nft1 with job batch system ID: 466 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:33:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmprnju2i9l/worker_log.txt +[2020-11-27T16:33:11-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-yb02nft1 +[2020-11-27T16:33:11-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-p55975qr with job batch system ID: 467 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:33:12-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpxx2tsg45/worker_log.txt +[2020-11-27T16:33:12-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-p55975qr +[2020-11-27T16:33:12-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-h7gmbwkp with job batch system ID: 468 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:33:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp4c14eq8l/worker_log.txt +[2020-11-27T16:33:13-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-h7gmbwkp +[2020-11-27T16:33:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:33:50: Ran radia on TEST:chr6 successfully +[2020-11-27T16:33:51-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-h5aeqrpp +[2020-11-27T16:33:51-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-_vedrzk2 with job batch system ID: 469 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:33:52-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp695wcen_/worker_log.txt +[2020-11-27T16:33:52-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-_vedrzk2 +[2020-11-27T16:33:52-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-b408bv9p with job batch system ID: 470 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:33:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdnoca5i6/worker_log.txt +[2020-11-27T16:35:30-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:35:30: Exporting files/for-job/kind-run_filter_radia/instance-b408bv9p/file-2b3shs37/chr6.vcf to output location +[2020-11-27T16:35:30-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:35:30: Ran filter-radia on TEST:chr6 successfully +[2020-11-27T16:35:30-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:35:30: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-_vedrzk2/cleanup/file-nitx_ux5/stream used 123.23% (6.5 GB [7019925504B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:35:31-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-b408bv9p +[2020-11-27T16:35:31-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-_1hb0opb with job batch system ID: 471 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:35:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7j8moz3z/worker_log.txt +[2020-11-27T16:35:32-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-_1hb0opb +[2020-11-27T16:35:32-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-_vedrzk2 with job batch system ID: 472 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:35:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp2a9hhsfr/worker_log.txt +[2020-11-27T16:35:32-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-_vedrzk2 +[2020-11-27T16:35:32-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-36ghujog with job batch system ID: 473 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:35:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmppdbtaz3a/worker_log.txt +[2020-11-27T16:35:33-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-36ghujog +[2020-11-27T16:36:35-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:36:35: Ran radia on TEST:chr5 successfully +[2020-11-27T16:36:35-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-vm3n1_ax +[2020-11-27T16:36:35-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-_gu2v2i2 with job batch system ID: 474 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:36:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpulwft_hy/worker_log.txt +[2020-11-27T16:36:36-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-_gu2v2i2 +[2020-11-27T16:36:36-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-jqgn_f6q with job batch system ID: 475 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:36:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpc2dn211t/worker_log.txt +[2020-11-27T16:38:18-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:38:18: Exporting files/for-job/kind-run_filter_radia/instance-jqgn_f6q/file-qh1vcmvs/chr5.vcf to output location +[2020-11-27T16:38:18-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:38:18: Ran filter-radia on TEST:chr5 successfully +[2020-11-27T16:38:18-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:38:18: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-_gu2v2i2/cleanup/file-l113us0_/stream used 123.26% (6.5 GB [7021936640B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:38:19-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-jqgn_f6q +[2020-11-27T16:38:19-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-z3wqswwv with job batch system ID: 476 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:38:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpo88rjavz/worker_log.txt +[2020-11-27T16:38:20-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-z3wqswwv +[2020-11-27T16:38:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-_gu2v2i2 with job batch system ID: 477 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:38:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdy56sbgl/worker_log.txt +[2020-11-27T16:38:21-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-_gu2v2i2 +[2020-11-27T16:38:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-5iaf3waa with job batch system ID: 478 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:38:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmphr6i09t0/worker_log.txt +[2020-11-27T16:38:22-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-5iaf3waa +[2020-11-27T16:38:46-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:38:46: Ran radia on TEST:chr4 successfully +[2020-11-27T16:38:47-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-tzskrbrh +[2020-11-27T16:38:47-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-v1uhafcn with job batch system ID: 479 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:38:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpttzlanr9/worker_log.txt +[2020-11-27T16:38:48-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-v1uhafcn +[2020-11-27T16:38:48-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-alluxza3 with job batch system ID: 480 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:38:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpaaeygwj3/worker_log.txt +[2020-11-27T16:39:22-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:39:22: Ran radia on TEST:chr7 successfully +[2020-11-27T16:39:23-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-883erw0y +[2020-11-27T16:39:23-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-sodt4ffx with job batch system ID: 481 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:39:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgbe9pnyz/worker_log.txt +[2020-11-27T16:39:24-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-sodt4ffx +[2020-11-27T16:39:24-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-zuw6zavq with job batch system ID: 482 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:39:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmppbgr_yt8/worker_log.txt +[2020-11-27T16:40:32-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:40:32: Exporting files/for-job/kind-run_filter_radia/instance-alluxza3/file-36o43q7b/chr4.vcf to output location +[2020-11-27T16:40:32-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:40:32: Ran filter-radia on TEST:chr4 successfully +[2020-11-27T16:40:32-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:40:32: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-v1uhafcn/cleanup/file-k9n6mx4p/stream used 123.32% (6.5 GB [7025307648B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:40:33-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-alluxza3 +[2020-11-27T16:40:33-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-wndj5dmt with job batch system ID: 483 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:40:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7phpyf6d/worker_log.txt +[2020-11-27T16:40:34-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-wndj5dmt +[2020-11-27T16:40:34-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-v1uhafcn with job batch system ID: 484 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:40:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmphtxydfxe/worker_log.txt +[2020-11-27T16:40:35-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-v1uhafcn +[2020-11-27T16:40:35-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-097yf3t5 with job batch system ID: 485 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:40:36-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpg03z3x0h/worker_log.txt +[2020-11-27T16:40:36-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-097yf3t5 +[2020-11-27T16:41:06-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:41:06: Exporting files/for-job/kind-run_filter_radia/instance-zuw6zavq/file-0_jvcpti/chr7.vcf to output location +[2020-11-27T16:41:06-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:41:06: Ran filter-radia on TEST:chr7 successfully +[2020-11-27T16:41:06-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:41:06: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-sodt4ffx/cleanup/file-6jk29xm8/stream used 123.20% (6.5 GB [7018770432B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:41:07-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-zuw6zavq +[2020-11-27T16:41:07-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-75en8g_a with job batch system ID: 486 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:41:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp9efb9jwe/worker_log.txt +[2020-11-27T16:41:07-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-75en8g_a +[2020-11-27T16:41:07-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-sodt4ffx with job batch system ID: 487 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:41:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpvdyr5yxa/worker_log.txt +[2020-11-27T16:41:08-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-sodt4ffx +[2020-11-27T16:41:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-hvhruste with job batch system ID: 488 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:41:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpqubc388j/worker_log.txt +[2020-11-27T16:41:09-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-hvhruste +[2020-11-27T16:41:43-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:41:43: Ran radia on TEST:chr3 successfully +[2020-11-27T16:41:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-8wsr28eg +[2020-11-27T16:41:44-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-k62g4z4a with job batch system ID: 489 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:41:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpu7_24ed4/worker_log.txt +[2020-11-27T16:41:45-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-k62g4z4a +[2020-11-27T16:41:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-3rs1r8fh with job batch system ID: 490 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:41:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpvq8gf7h1/worker_log.txt +[2020-11-27T16:43:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:43:37: Exporting files/for-job/kind-run_filter_radia/instance-3rs1r8fh/file-7yeabgpp/chr3.vcf to output location +[2020-11-27T16:43:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:43:37: Ran filter-radia on TEST:chr3 successfully +[2020-11-27T16:43:37-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:43:37: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-k62g4z4a/cleanup/file-1y2ewvxw/stream used 123.23% (6.5 GB [7020027904B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:43:38-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-3rs1r8fh +[2020-11-27T16:43:38-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-t7ds7prc with job batch system ID: 491 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:43:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp279x5quo/worker_log.txt +[2020-11-27T16:43:39-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-t7ds7prc +[2020-11-27T16:43:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-k62g4z4a with job batch system ID: 492 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:43:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp577511tm/worker_log.txt +[2020-11-27T16:43:40-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-k62g4z4a +[2020-11-27T16:43:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-hht82wzu with job batch system ID: 493 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:43:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1harlsq9/worker_log.txt +[2020-11-27T16:43:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-hht82wzu +[2020-11-27T16:49:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:49:15: Ran radia on TEST:chr1 successfully +[2020-11-27T16:49:15-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-h2czw2cf +[2020-11-27T16:49:15-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-cdvn1zdo with job batch system ID: 494 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:49:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp653ytk_u/worker_log.txt +[2020-11-27T16:49:17-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-cdvn1zdo +[2020-11-27T16:49:17-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-xi09b03_ with job batch system ID: 495 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:49:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:49:17: Ran radia on TEST:chr2 successfully +[2020-11-27T16:49:17-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpp6ghlwu_/worker_log.txt +[2020-11-27T16:49:18-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-run_radia_perchrom/instance-cax7nt5c +[2020-11-27T16:49:18-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-c78wworc with job batch system ID: 496 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:49:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_f1do6gy/worker_log.txt +[2020-11-27T16:49:19-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-c78wworc +[2020-11-27T16:49:19-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-run_filter_radia/instance-9cctaeny with job batch system ID: 497 and cores: 1, disk: 5.3 G, and memory: 6.0 G +[2020-11-27T16:49:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbmsarq0e/worker_log.txt +[2020-11-27T16:51:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:51:15: Exporting files/for-job/kind-run_filter_radia/instance-xi09b03_/file-e7nlbmby/chr1.vcf to output location +[2020-11-27T16:51:15-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:51:15: Ran filter-radia on TEST:chr1 successfully +[2020-11-27T16:51:15-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:51:15: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-cdvn1zdo/cleanup/file-gwaux94b/stream used 123.27% (6.5 GB [7022583808B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:51:16-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-xi09b03_ +[2020-11-27T16:51:16-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-4liq2u6s with job batch system ID: 498 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:51:17-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp216rjscx/worker_log.txt +[2020-11-27T16:51:17-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-4liq2u6s +[2020-11-27T16:51:17-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-cdvn1zdo with job batch system ID: 499 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:51:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpys8q7ahb/worker_log.txt +[2020-11-27T16:51:18-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-cdvn1zdo +[2020-11-27T16:51:18-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-zc4ncec4 with job batch system ID: 500 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:51:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmppewghzhb/worker_log.txt +[2020-11-27T16:51:18-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-zc4ncec4 +[2020-11-27T16:51:21-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:51:21: Exporting files/for-job/kind-run_filter_radia/instance-9cctaeny/file-9b4z4yhy/chr2.vcf to output location +[2020-11-27T16:51:21-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:51:21: Ran filter-radia on TEST:chr2 successfully +[2020-11-27T16:51:21-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 16:51:21: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-Job/instance-c78wworc/cleanup/file-malz08jd/stream used 123.24% (6.5 GB [7020883968B] used, 5.3 GB [5696827974B] requested) at the end of its run. +[2020-11-27T16:51:22-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-run_filter_radia/instance-9cctaeny +[2020-11-27T16:51:22-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-086efq14 with job batch system ID: 501 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:51:23-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpln8b0pnb/worker_log.txt +[2020-11-27T16:51:23-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-086efq14 +[2020-11-27T16:51:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_filter_radia' kind-Job/instance-c78wworc with job batch system ID: 502 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:51:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpf2v72xuy/worker_log.txt +[2020-11-27T16:51:24-0800] [MainThread] [I] [toil.leader] Job ended: 'run_filter_radia' kind-Job/instance-c78wworc +[2020-11-27T16:51:24-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia_perchrom' kind-EncapsulatedJob/instance-tqnu8l56 with job batch system ID: 503 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T16:51:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdimk6cxz/worker_log.txt +[2020-11-27T16:51:25-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia_perchrom' kind-EncapsulatedJob/instance-tqnu8l56 +[2020-11-27T16:51:25-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-7rcs33og with job batch system ID: 504 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T16:51:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp29tma380/worker_log.txt +[2020-11-27T16:51:26-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-7rcs33og +[2020-11-27T17:11:40-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run diff --git a/errors/2020-11-27-17-28-31.txt b/errors/2020-11-27-17-28-31.txt new file mode 100644 index 00000000..c13d68d4 --- /dev/null +++ b/errors/2020-11-27-17-28-31.txt @@ -0,0 +1,103 @@ +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-y2fcywgt +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-get_patient_fastqs/instance-2pjao3y3 +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-5zdtbtsn +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-gxt8is74 +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-prepare_samples/instance-qyxpumun +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-5i9__ll1 +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ey78_apt +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-ixd9le0x +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-Job/instance-19ln8xui +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r0a3rxak +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xqryz_ub +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-czlzbjjr +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-Job/instance-p8u5n56k +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-m6rpu3gh +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-t15qgjy0 +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-5drwan9s +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-qi2fqyz_ +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-p8mcfs96 +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-9i484bqi +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ntmj9aw5 +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-f3zgk1bv +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-mgq6ru10 +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-gyttywxo +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-1jh6nqug +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-h0tdg49o +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-tm0pcusm +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-_u307ivu +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-du24kqrt +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-Job/instance-p3i6d1f0 +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-mgdq08hc +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-y0q8img4 +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-get_patient_fastqs/instance-0csdd7bl +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-4f2ha64d +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-989tgher +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-e1yejtsm +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-wlu1vdpy +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-o9p20i2e +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-Job/instance-hlsgfes5 +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-hxsjhjvp +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-fgj0w2_w +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-Job/instance-5x2sz54b +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-Job/instance-7s1jyb6z +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-i6wy4k24 +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-get_patient_fastqs/instance-ttmsgsd1 +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-p8idulz4 +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-Job/instance-l6jndejt +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-77hux1xy +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-Job/instance-j3z8ggs3 +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-jvmeok8w +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-y_lo_bvx +[2020-11-27T17:28:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-Job/instance-wizon1te +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5zdtbtsn with job batch system ID: 0 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-t15qgjy0 with job batch system ID: 1 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-989tgher with job batch system ID: 2 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fgj0w2_w with job batch system ID: 3 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ey78_apt with job batch system ID: 4 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-4f2ha64d with job batch system ID: 5 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-1jh6nqug with job batch system ID: 6 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-y2fcywgt with job batch system ID: 7 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-f3zgk1bv with job batch system ID: 8 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r0a3rxak with job batch system ID: 9 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9i484bqi with job batch system ID: 10 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h0tdg49o with job batch system ID: 11 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-hxsjhjvp with job batch system ID: 12 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ntmj9aw5 with job batch system ID: 13 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-e1yejtsm with job batch system ID: 14 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-_u307ivu with job batch system ID: 15 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-du24kqrt with job batch system ID: 16 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5i9__ll1 with job batch system ID: 17 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mgq6ru10 with job batch system ID: 18 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-p8mcfs96 with job batch system ID: 19 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-77hux1xy with job batch system ID: 20 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-m6rpu3gh with job batch system ID: 21 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gyttywxo with job batch system ID: 22 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jvmeok8w with job batch system ID: 23 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xqryz_ub with job batch system ID: 24 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpa5ch08pe/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpoz4hy9nf/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp5dfx2vnj/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp8ldhz0h5/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpr121kz97/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpfbzfk4er/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpa8bpemfz/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp9kntrp64/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp2chvs32p/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_gvfmm2j/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpav4t4kw9/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpsb2sduaw/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbv9zmlk5/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgmomol0q/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmprn6pyoxg/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_4k6ulxv/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp9g2sdm8h/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmph1yrm09z/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpftk5m423/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp64l7484p/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_mx8o_9l/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7bvow4pq/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_dd9dmu1/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpo7_f9s0i/worker_log.txt +[2020-11-27T17:28:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp2waq96q2/worker_log.txt +[2020-11-27T17:28:35-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run diff --git a/errors/2020-11-27-17-32-35.txt b/errors/2020-11-27-17-32-35.txt new file mode 100644 index 00000000..57253c5d --- /dev/null +++ b/errors/2020-11-27-17-32-35.txt @@ -0,0 +1,52 @@ +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5zdtbtsn with job batch system ID: 0 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-t15qgjy0 with job batch system ID: 1 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-989tgher with job batch system ID: 2 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fgj0w2_w with job batch system ID: 3 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ey78_apt with job batch system ID: 4 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-4f2ha64d with job batch system ID: 5 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-1jh6nqug with job batch system ID: 6 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-y2fcywgt with job batch system ID: 7 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-f3zgk1bv with job batch system ID: 8 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r0a3rxak with job batch system ID: 9 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9i484bqi with job batch system ID: 10 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h0tdg49o with job batch system ID: 11 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-hxsjhjvp with job batch system ID: 12 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ntmj9aw5 with job batch system ID: 13 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-e1yejtsm with job batch system ID: 14 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-_u307ivu with job batch system ID: 15 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-du24kqrt with job batch system ID: 16 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5i9__ll1 with job batch system ID: 17 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mgq6ru10 with job batch system ID: 18 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-p8mcfs96 with job batch system ID: 19 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-77hux1xy with job batch system ID: 20 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-m6rpu3gh with job batch system ID: 21 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gyttywxo with job batch system ID: 22 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jvmeok8w with job batch system ID: 23 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xqryz_ub with job batch system ID: 24 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:32:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_g_bhupd/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp66pwuqty/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7zy_2hxf/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_4wxtk1k/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmppl7d4z33/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpu39pk797/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp20hd1e3w/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpneyys0o3/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpuy5wl84j/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpl_q5ejqa/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_lopi2k8/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpld08m561/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpnr9wd5ev/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpb2idjsto/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp5uwdoqld/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmptcuzwcoc/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp3acx3l6h/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpib515rhr/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpomu0aaf2/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpf562bkle/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmph0ky5ufv/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmphiwolsa0/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpa14f6v19/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpro45a1dt/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdncaiqrq/worker_log.txt +[2020-11-27T17:32:39-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run diff --git a/errors/2020-11-27-17-47-44.txt b/errors/2020-11-27-17-47-44.txt new file mode 100644 index 00000000..f4e4c324 --- /dev/null +++ b/errors/2020-11-27-17-47-44.txt @@ -0,0 +1,52 @@ +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5zdtbtsn with job batch system ID: 0 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-t15qgjy0 with job batch system ID: 1 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-989tgher with job batch system ID: 2 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fgj0w2_w with job batch system ID: 3 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ey78_apt with job batch system ID: 4 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-4f2ha64d with job batch system ID: 5 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-1jh6nqug with job batch system ID: 6 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-y2fcywgt with job batch system ID: 7 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-f3zgk1bv with job batch system ID: 8 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r0a3rxak with job batch system ID: 9 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9i484bqi with job batch system ID: 10 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h0tdg49o with job batch system ID: 11 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-hxsjhjvp with job batch system ID: 12 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ntmj9aw5 with job batch system ID: 13 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-e1yejtsm with job batch system ID: 14 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-_u307ivu with job batch system ID: 15 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-du24kqrt with job batch system ID: 16 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5i9__ll1 with job batch system ID: 17 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mgq6ru10 with job batch system ID: 18 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-p8mcfs96 with job batch system ID: 19 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-77hux1xy with job batch system ID: 20 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-m6rpu3gh with job batch system ID: 21 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gyttywxo with job batch system ID: 22 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jvmeok8w with job batch system ID: 23 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xqryz_ub with job batch system ID: 24 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpy39ixmft/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp8hk5hgr3/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpqpckrunc/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpb7sqglou/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpeudwcvv0/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp9rp2zs47/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpanfu4ojy/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbe7wnwzt/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmps87z28mj/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpitxmzq2n/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmphppaxt6e/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_rfwa_wp/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpe01ne95j/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpw1mkl8u6/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp6_9pijev/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpebcka437/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpc7y01v5y/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgqgyqlny/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpk9z086tm/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpq94ftwid/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbkm8cl53/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp8a3bzbmv/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmps1xdgxis/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpz_n_wcy6/worker_log.txt +[2020-11-27T17:47:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpj6ucr3dw/worker_log.txt +[2020-11-27T17:47:48-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run diff --git a/errors/2020-11-27-18-37-04.txt b/errors/2020-11-27-18-37-04.txt new file mode 100644 index 00000000..47a264c1 --- /dev/null +++ b/errors/2020-11-27-18-37-04.txt @@ -0,0 +1,52 @@ +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5zdtbtsn with job batch system ID: 0 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-t15qgjy0 with job batch system ID: 1 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-989tgher with job batch system ID: 2 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fgj0w2_w with job batch system ID: 3 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ey78_apt with job batch system ID: 4 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-4f2ha64d with job batch system ID: 5 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-1jh6nqug with job batch system ID: 6 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-y2fcywgt with job batch system ID: 7 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-f3zgk1bv with job batch system ID: 8 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r0a3rxak with job batch system ID: 9 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9i484bqi with job batch system ID: 10 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h0tdg49o with job batch system ID: 11 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-hxsjhjvp with job batch system ID: 12 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ntmj9aw5 with job batch system ID: 13 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-e1yejtsm with job batch system ID: 14 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-_u307ivu with job batch system ID: 15 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-du24kqrt with job batch system ID: 16 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5i9__ll1 with job batch system ID: 17 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mgq6ru10 with job batch system ID: 18 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-p8mcfs96 with job batch system ID: 19 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-77hux1xy with job batch system ID: 20 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-m6rpu3gh with job batch system ID: 21 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gyttywxo with job batch system ID: 22 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jvmeok8w with job batch system ID: 23 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:06-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xqryz_ub with job batch system ID: 24 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:37:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmponwrq0r3/worker_log.txt +[2020-11-27T18:37:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpe99r0rnx/worker_log.txt +[2020-11-27T18:37:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpjf_xwzmw/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpsy025hmm/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmptg3jm18s/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmppkd2f9iu/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpvrid9nhs/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpuyg97a7f/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpj7y5t1_u/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp3yle7djj/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbrbagd17/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp90nhepuu/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpn8x1grlg/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpzjc4rkmn/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp0rrv8ksa/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp10i8nvon/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmps0sjibb3/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp4w78rfvp/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpsto5xtdo/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpfsfvtf7a/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpr9oazzoj/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp9wd0440t/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp64i8jdou/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpk7zhyr5e/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbvcsty3r/worker_log.txt +[2020-11-27T18:37:08-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run diff --git a/errors/2020-11-27-18-42-33.txt b/errors/2020-11-27-18-42-33.txt new file mode 100644 index 00000000..e92d4496 --- /dev/null +++ b/errors/2020-11-27-18-42-33.txt @@ -0,0 +1,1577 @@ +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5zdtbtsn with job batch system ID: 0 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-t15qgjy0 with job batch system ID: 1 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-989tgher with job batch system ID: 2 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fgj0w2_w with job batch system ID: 3 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ey78_apt with job batch system ID: 4 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-4f2ha64d with job batch system ID: 5 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-1jh6nqug with job batch system ID: 6 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-y2fcywgt with job batch system ID: 7 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-f3zgk1bv with job batch system ID: 8 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r0a3rxak with job batch system ID: 9 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9i484bqi with job batch system ID: 10 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h0tdg49o with job batch system ID: 11 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-hxsjhjvp with job batch system ID: 12 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ntmj9aw5 with job batch system ID: 13 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-e1yejtsm with job batch system ID: 14 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-_u307ivu with job batch system ID: 15 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-du24kqrt with job batch system ID: 16 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5i9__ll1 with job batch system ID: 17 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mgq6ru10 with job batch system ID: 18 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-p8mcfs96 with job batch system ID: 19 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-77hux1xy with job batch system ID: 20 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-m6rpu3gh with job batch system ID: 21 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gyttywxo with job batch system ID: 22 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jvmeok8w with job batch system ID: 23 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:30-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xqryz_ub with job batch system ID: 24 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:45:31-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpzg5zzrmk/worker_log.txt +[2020-11-27T18:45:31-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp9jdeea1n/worker_log.txt +[2020-11-27T18:45:31-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpb2l863hh/worker_log.txt +[2020-11-27T18:45:32-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-27T18:45:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp58_1cn9_/worker_log.txt +[2020-11-27T18:45:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpab800ddd/worker_log.txt +[2020-11-27T18:46:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpitet8ox3/worker_log.txt +[2020-11-27T18:46:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp3be7x0u9/worker_log.txt +[2020-11-27T18:46:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpjcwj2nc6/worker_log.txt +[2020-11-27T18:46:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpv8bf72q4/worker_log.txt +[2020-11-27T18:46:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp6yhcrhtb/worker_log.txt +[2020-11-27T18:46:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp2teuozjy/worker_log.txt +[2020-11-27T18:46:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpjh9fin3x/worker_log.txt +[2020-11-27T18:46:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpansptsae/worker_log.txt +[2020-11-27T18:46:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp2p9cmaq3/worker_log.txt +[2020-11-27T18:46:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpg27akjz0/worker_log.txt +[2020-11-27T18:46:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp18xq_ze0/worker_log.txt +[2020-11-27T18:46:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpz2u3k3m2/worker_log.txt +[2020-11-27T18:46:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpuydnybq9/worker_log.txt +[2020-11-27T18:46:58-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpmzhqf2mq/worker_log.txt +[2020-11-27T18:47:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpq7fv1wvf/worker_log.txt +[2020-11-27T18:47:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpiackf7oz/worker_log.txt +[2020-11-27T18:47:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpddobhrbc/worker_log.txt +[2020-11-27T18:47:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpopxybrb0/worker_log.txt +[2020-11-27T18:47:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp88s1wb0r/worker_log.txt +[2020-11-27T18:47:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpz_r7nf7h/worker_log.txt +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-gyttywxo. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gyttywxo +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gyttywxo +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-gyttywxo follows: +=========> + [2020-11-27T18:46:58-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:46:58-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:47:00-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gyttywxo with ID kind-run_mutect_perchrom/instance-gyttywxo to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gyttywxo with job batch system ID: 25 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-t15qgjy0. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-t15qgjy0 +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-t15qgjy0 +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-t15qgjy0 follows: +=========> + [2020-11-27T18:45:31-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:45:31-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:47:00-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-t15qgjy0 with ID kind-run_mutect_perchrom/instance-t15qgjy0 to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-t15qgjy0 with job batch system ID: 26 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-5zdtbtsn. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5zdtbtsn +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5zdtbtsn +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-5zdtbtsn follows: +=========> + [2020-11-27T18:47:00-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:47:00-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5zdtbtsn with ID kind-run_mutect_perchrom/instance-5zdtbtsn to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5zdtbtsn with job batch system ID: 27 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-fgj0w2_w. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fgj0w2_w +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fgj0w2_w +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-fgj0w2_w follows: +=========> + [2020-11-27T18:45:31-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:45:31-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fgj0w2_w with ID kind-run_mutect_perchrom/instance-fgj0w2_w to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fgj0w2_w with job batch system ID: 28 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-f3zgk1bv. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-f3zgk1bv +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-f3zgk1bv +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-f3zgk1bv follows: +=========> + [2020-11-27T18:45:31-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:45:31-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-f3zgk1bv with ID kind-run_mutect_perchrom/instance-f3zgk1bv to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-f3zgk1bv with job batch system ID: 29 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-hxsjhjvp. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-hxsjhjvp +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-hxsjhjvp +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-hxsjhjvp follows: +=========> + [2020-11-27T18:46:44-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:46:44-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:47:00-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-hxsjhjvp with ID kind-run_mutect_perchrom/instance-hxsjhjvp to 1 +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-ntmj9aw5. +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-du24kqrt. +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-hxsjhjvp with job batch system ID: 30 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ntmj9aw5 +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ntmj9aw5 +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-ntmj9aw5 follows: +=========> + [2020-11-27T18:46:58-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:46:58-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ntmj9aw5 with ID kind-run_mutect_perchrom/instance-ntmj9aw5 to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ntmj9aw5 with job batch system ID: 31 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-du24kqrt +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-du24kqrt +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-du24kqrt follows: +=========> + [2020-11-27T18:46:58-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:46:58-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-du24kqrt with ID kind-run_mutect_perchrom/instance-du24kqrt to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-du24kqrt with job batch system ID: 32 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-4f2ha64d. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-4f2ha64d +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-4f2ha64d +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-4f2ha64d follows: +=========> + [2020-11-27T18:45:41-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:45:41-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-4f2ha64d with ID kind-run_mutect_perchrom/instance-4f2ha64d to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-4f2ha64d with job batch system ID: 33 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-mgq6ru10. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mgq6ru10 +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mgq6ru10 +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-mgq6ru10 follows: +=========> + [2020-11-27T18:46:44-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:46:44-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:47:00-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mgq6ru10 with ID kind-run_mutect_perchrom/instance-mgq6ru10 to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mgq6ru10 with job batch system ID: 34 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-5i9__ll1. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5i9__ll1 +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5i9__ll1 +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-5i9__ll1 follows: +=========> + [2020-11-27T18:45:38-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:45:38-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5i9__ll1 with ID kind-run_mutect_perchrom/instance-5i9__ll1 to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5i9__ll1 with job batch system ID: 35 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-m6rpu3gh. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-m6rpu3gh +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-m6rpu3gh +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-m6rpu3gh follows: +=========> + [2020-11-27T18:46:58-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:46:58-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-m6rpu3gh with ID kind-run_mutect_perchrom/instance-m6rpu3gh to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-m6rpu3gh with job batch system ID: 36 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-989tgher. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-989tgher +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-989tgher +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-989tgher follows: +=========> + [2020-11-27T18:46:58-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:46:58-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-989tgher with ID kind-run_mutect_perchrom/instance-989tgher to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-989tgher with job batch system ID: 37 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-ey78_apt. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ey78_apt +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ey78_apt +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-ey78_apt follows: +=========> + [2020-11-27T18:47:00-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:47:00-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ey78_apt with ID kind-run_mutect_perchrom/instance-ey78_apt to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ey78_apt with job batch system ID: 38 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-1jh6nqug. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-1jh6nqug +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-1jh6nqug +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-1jh6nqug follows: +=========> + [2020-11-27T18:46:44-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:46:44-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-1jh6nqug with ID kind-run_mutect_perchrom/instance-1jh6nqug to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-1jh6nqug with job batch system ID: 39 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-y2fcywgt. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-y2fcywgt +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-y2fcywgt +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-y2fcywgt follows: +=========> + [2020-11-27T18:46:58-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:46:58-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-y2fcywgt with ID kind-run_mutect_perchrom/instance-y2fcywgt to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-y2fcywgt with job batch system ID: 40 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-r0a3rxak. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r0a3rxak +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r0a3rxak +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-r0a3rxak follows: +=========> + [2020-11-27T18:46:58-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:46:58-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r0a3rxak with ID kind-run_mutect_perchrom/instance-r0a3rxak to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r0a3rxak with job batch system ID: 41 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-9i484bqi. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9i484bqi +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9i484bqi +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-9i484bqi follows: +=========> + [2020-11-27T18:47:00-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:47:00-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9i484bqi with ID kind-run_mutect_perchrom/instance-9i484bqi to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9i484bqi with job batch system ID: 42 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-h0tdg49o. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h0tdg49o +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h0tdg49o +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-h0tdg49o follows: +=========> + [2020-11-27T18:47:00-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:47:00-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h0tdg49o with ID kind-run_mutect_perchrom/instance-h0tdg49o to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h0tdg49o with job batch system ID: 43 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-e1yejtsm. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-e1yejtsm +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-e1yejtsm +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-e1yejtsm follows: +=========> + [2020-11-27T18:46:58-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:46:58-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-e1yejtsm with ID kind-run_mutect_perchrom/instance-e1yejtsm to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-e1yejtsm with job batch system ID: 44 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-_u307ivu. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-_u307ivu +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-_u307ivu +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-_u307ivu follows: +=========> + [2020-11-27T18:47:00-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:47:00-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-_u307ivu with ID kind-run_mutect_perchrom/instance-_u307ivu to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-_u307ivu with job batch system ID: 45 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-p8mcfs96. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-p8mcfs96 +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-p8mcfs96 +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-p8mcfs96 follows: +=========> + [2020-11-27T18:46:51-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:46:51-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-p8mcfs96 with ID kind-run_mutect_perchrom/instance-p8mcfs96 to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-p8mcfs96 with job batch system ID: 46 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-77hux1xy. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-77hux1xy +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-77hux1xy +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-77hux1xy follows: +=========> + [2020-11-27T18:47:00-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:47:00-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-77hux1xy with ID kind-run_mutect_perchrom/instance-77hux1xy to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-77hux1xy with job batch system ID: 47 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-jvmeok8w. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jvmeok8w +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jvmeok8w +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-jvmeok8w follows: +=========> + [2020-11-27T18:46:44-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:46:44-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jvmeok8w with ID kind-run_mutect_perchrom/instance-jvmeok8w to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jvmeok8w with job batch system ID: 48 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:10-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-xqryz_ub. +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xqryz_ub +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xqryz_ub +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-xqryz_ub follows: +=========> + [2020-11-27T18:46:18-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:46:18-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:10-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:10-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:10-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xqryz_ub with ID kind-run_mutect_perchrom/instance-xqryz_ub to 1 +[2020-11-27T18:48:10-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xqryz_ub with job batch system ID: 49 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpuqjou1fu/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpnb8idjh_/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpcpsy_vim/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmparc4qchx/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpowrncml8/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpj06jbw9k/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpg_dqce4n/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpt5m8c30o/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpp3iunvrq/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgbj6k8qt/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpt9ho6_p0/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpt_sm4j29/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmprumpyk60/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp3sna27gi/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1dqkslu_/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp2xevqw_w/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgfebcaby/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpgmi8qsz8/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpkzikfebz/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1duuxvs2/worker_log.txt +[2020-11-27T18:48:11-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-5zdtbtsn. +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5zdtbtsn +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5zdtbtsn +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-5zdtbtsn follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:11-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5zdtbtsn with ID kind-run_mutect_perchrom/instance-5zdtbtsn to 0 +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5zdtbtsn with ID kind-run_mutect_perchrom/instance-5zdtbtsn is completely failed +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpoqlzkl3f/worker_log.txt +[2020-11-27T18:48:11-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-gyttywxo. +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gyttywxo +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gyttywxo +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-gyttywxo follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:11-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gyttywxo with ID kind-run_mutect_perchrom/instance-gyttywxo to 0 +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gyttywxo with ID kind-run_mutect_perchrom/instance-gyttywxo is completely failed +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpaagledfa/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpysb37pv2/worker_log.txt +[2020-11-27T18:48:11-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-t15qgjy0. +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-t15qgjy0 +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-t15qgjy0 +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-t15qgjy0 follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:11-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-t15qgjy0 with ID kind-run_mutect_perchrom/instance-t15qgjy0 to 0 +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-t15qgjy0 with ID kind-run_mutect_perchrom/instance-t15qgjy0 is completely failed +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpnefh6elt/worker_log.txt +[2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpe7ufve49/worker_log.txt +[2020-11-27T18:48:11-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-fgj0w2_w. +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fgj0w2_w +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fgj0w2_w +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-fgj0w2_w follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:11-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fgj0w2_w with ID kind-run_mutect_perchrom/instance-fgj0w2_w to 0 +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fgj0w2_w with ID kind-run_mutect_perchrom/instance-fgj0w2_w is completely failed +[2020-11-27T18:48:11-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-f3zgk1bv. +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-f3zgk1bv +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-f3zgk1bv +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-f3zgk1bv follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:11-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-f3zgk1bv with ID kind-run_mutect_perchrom/instance-f3zgk1bv to 0 +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-f3zgk1bv with ID kind-run_mutect_perchrom/instance-f3zgk1bv is completely failed +[2020-11-27T18:48:11-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-hxsjhjvp. +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-hxsjhjvp +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-hxsjhjvp +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-hxsjhjvp follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:11-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-hxsjhjvp with ID kind-run_mutect_perchrom/instance-hxsjhjvp to 0 +[2020-11-27T18:48:11-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-ntmj9aw5. +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-hxsjhjvp with ID kind-run_mutect_perchrom/instance-hxsjhjvp is completely failed +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ntmj9aw5 +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ntmj9aw5 +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-ntmj9aw5 follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:11-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ntmj9aw5 with ID kind-run_mutect_perchrom/instance-ntmj9aw5 to 0 +[2020-11-27T18:48:11-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-du24kqrt. +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ntmj9aw5 with ID kind-run_mutect_perchrom/instance-ntmj9aw5 is completely failed +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-du24kqrt +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-du24kqrt +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-du24kqrt follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:11-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-du24kqrt with ID kind-run_mutect_perchrom/instance-du24kqrt to 0 +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-du24kqrt with ID kind-run_mutect_perchrom/instance-du24kqrt is completely failed +[2020-11-27T18:48:11-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-4f2ha64d. +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-4f2ha64d +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-4f2ha64d +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-4f2ha64d follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:11-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-4f2ha64d with ID kind-run_mutect_perchrom/instance-4f2ha64d to 0 +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-4f2ha64d with ID kind-run_mutect_perchrom/instance-4f2ha64d is completely failed +[2020-11-27T18:48:11-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-mgq6ru10. +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mgq6ru10 +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mgq6ru10 +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-mgq6ru10 follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:11-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mgq6ru10 with ID kind-run_mutect_perchrom/instance-mgq6ru10 to 0 +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mgq6ru10 with ID kind-run_mutect_perchrom/instance-mgq6ru10 is completely failed +[2020-11-27T18:48:11-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-5i9__ll1. +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5i9__ll1 +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5i9__ll1 +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-5i9__ll1 follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:11-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5i9__ll1 with ID kind-run_mutect_perchrom/instance-5i9__ll1 to 0 +[2020-11-27T18:48:11-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5i9__ll1 with ID kind-run_mutect_perchrom/instance-5i9__ll1 is completely failed +[2020-11-27T18:48:12-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-ey78_apt. +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ey78_apt +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ey78_apt +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-ey78_apt follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:11-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ey78_apt with ID kind-run_mutect_perchrom/instance-ey78_apt to 0 +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ey78_apt with ID kind-run_mutect_perchrom/instance-ey78_apt is completely failed +[2020-11-27T18:48:12-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-_u307ivu. +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-_u307ivu +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-_u307ivu +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-_u307ivu follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:12-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:12-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-_u307ivu with ID kind-run_mutect_perchrom/instance-_u307ivu to 0 +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-_u307ivu with ID kind-run_mutect_perchrom/instance-_u307ivu is completely failed +[2020-11-27T18:48:12-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-e1yejtsm. +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-e1yejtsm +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-e1yejtsm +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-e1yejtsm follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:12-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:12-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-e1yejtsm with ID kind-run_mutect_perchrom/instance-e1yejtsm to 0 +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-e1yejtsm with ID kind-run_mutect_perchrom/instance-e1yejtsm is completely failed +[2020-11-27T18:48:12-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-989tgher. +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-989tgher +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-989tgher +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-989tgher follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:12-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:12-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-989tgher with ID kind-run_mutect_perchrom/instance-989tgher to 0 +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-989tgher with ID kind-run_mutect_perchrom/instance-989tgher is completely failed +[2020-11-27T18:48:12-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-9i484bqi. +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9i484bqi +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9i484bqi +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-9i484bqi follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:12-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:12-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9i484bqi with ID kind-run_mutect_perchrom/instance-9i484bqi to 0 +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9i484bqi with ID kind-run_mutect_perchrom/instance-9i484bqi is completely failed +[2020-11-27T18:48:12-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-m6rpu3gh. +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-m6rpu3gh +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-m6rpu3gh +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-m6rpu3gh follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:12-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:12-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-m6rpu3gh with ID kind-run_mutect_perchrom/instance-m6rpu3gh to 0 +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-m6rpu3gh with ID kind-run_mutect_perchrom/instance-m6rpu3gh is completely failed +[2020-11-27T18:48:12-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-1jh6nqug. +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-1jh6nqug +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-1jh6nqug +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-1jh6nqug follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:12-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:12-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-1jh6nqug with ID kind-run_mutect_perchrom/instance-1jh6nqug to 0 +[2020-11-27T18:48:12-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-y2fcywgt. +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-1jh6nqug with ID kind-run_mutect_perchrom/instance-1jh6nqug is completely failed +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-y2fcywgt +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-y2fcywgt +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-y2fcywgt follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:12-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:12-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:12-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-r0a3rxak. +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-y2fcywgt with ID kind-run_mutect_perchrom/instance-y2fcywgt to 0 +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-y2fcywgt with ID kind-run_mutect_perchrom/instance-y2fcywgt is completely failed +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r0a3rxak +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r0a3rxak +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-r0a3rxak follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:12-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:12-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r0a3rxak with ID kind-run_mutect_perchrom/instance-r0a3rxak to 0 +[2020-11-27T18:48:12-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-h0tdg49o. +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r0a3rxak with ID kind-run_mutect_perchrom/instance-r0a3rxak is completely failed +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h0tdg49o +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h0tdg49o +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-h0tdg49o follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:12-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:12-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h0tdg49o with ID kind-run_mutect_perchrom/instance-h0tdg49o to 0 +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h0tdg49o with ID kind-run_mutect_perchrom/instance-h0tdg49o is completely failed +[2020-11-27T18:48:12-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-77hux1xy. +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-77hux1xy +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-77hux1xy +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-77hux1xy follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:12-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:12-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-77hux1xy with ID kind-run_mutect_perchrom/instance-77hux1xy to 0 +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-77hux1xy with ID kind-run_mutect_perchrom/instance-77hux1xy is completely failed +[2020-11-27T18:48:12-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-jvmeok8w. +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jvmeok8w +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jvmeok8w +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-jvmeok8w follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:12-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:12-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jvmeok8w with ID kind-run_mutect_perchrom/instance-jvmeok8w to 0 +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jvmeok8w with ID kind-run_mutect_perchrom/instance-jvmeok8w is completely failed +[2020-11-27T18:48:12-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-xqryz_ub. +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xqryz_ub +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xqryz_ub +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-xqryz_ub follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:12-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:12-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xqryz_ub with ID kind-run_mutect_perchrom/instance-xqryz_ub to 0 +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xqryz_ub with ID kind-run_mutect_perchrom/instance-xqryz_ub is completely failed +[2020-11-27T18:48:12-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_mutect_perchrom file:/scratch/drkthomp/jobStore kind-run_mutect_perchrom/instance-p8mcfs96. +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-p8mcfs96 +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-p8mcfs96 +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Log from job kind-run_mutect_perchrom/instance-p8mcfs96 follows: +=========> + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T18:48:11-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + [2020-11-27T18:48:12-0800] [MainThread] [I] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: unzipping dbsnp.vcf + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_calling/mutect.py", line 129, in run_mutect_perchrom + input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 179, in gunzip + shutil.copyfileobj(infile, outfile) + File "/private/home/drkthomp/protect3/venv/lib64/python3.6/shutil.py", line 82, in copyfileobj + fdst.write(buf) + TypeError: write() argument must be str, not bytes + [2020-11-27T18:48:12-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard + DEBUG: gunzip funciton + DEBUG: OPENING GZ FILE +<========= +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-p8mcfs96 with ID kind-run_mutect_perchrom/instance-p8mcfs96 to 0 +[2020-11-27T18:48:12-0800] [MainThread] [W] [toil.leader] Job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-p8mcfs96 with ID kind-run_mutect_perchrom/instance-p8mcfs96 is completely failed diff --git a/errors/2020-11-27-18-51-38.txt b/errors/2020-11-27-18-51-38.txt new file mode 100644 index 00000000..1d5e2081 --- /dev/null +++ b/errors/2020-11-27-18-51-38.txt @@ -0,0 +1,808 @@ +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-du24kqrt +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-p8mcfs96 +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-5i9__ll1 +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-989tgher +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-y2fcywgt +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ey78_apt +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-77hux1xy +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-r0a3rxak +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-e1yejtsm +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-gyttywxo +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-5zdtbtsn +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-m6rpu3gh +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-hxsjhjvp +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-t15qgjy0 +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-h0tdg49o +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-1jh6nqug +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-mgq6ru10 +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-ntmj9aw5 +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-fgj0w2_w +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-f3zgk1bv +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-9i484bqi +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-xqryz_ub +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-_u307ivu +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-jvmeok8w +[2020-11-27T18:51:40-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_mutect_perchrom/instance-4f2ha64d +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5zdtbtsn with job batch system ID: 0 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-t15qgjy0 with job batch system ID: 1 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-989tgher with job batch system ID: 2 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fgj0w2_w with job batch system ID: 3 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ey78_apt with job batch system ID: 4 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-4f2ha64d with job batch system ID: 5 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-1jh6nqug with job batch system ID: 6 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-y2fcywgt with job batch system ID: 7 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-f3zgk1bv with job batch system ID: 8 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r0a3rxak with job batch system ID: 9 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9i484bqi with job batch system ID: 10 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h0tdg49o with job batch system ID: 11 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-hxsjhjvp with job batch system ID: 12 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ntmj9aw5 with job batch system ID: 13 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-e1yejtsm with job batch system ID: 14 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-_u307ivu with job batch system ID: 15 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-du24kqrt with job batch system ID: 16 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5i9__ll1 with job batch system ID: 17 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mgq6ru10 with job batch system ID: 18 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-p8mcfs96 with job batch system ID: 19 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-77hux1xy with job batch system ID: 20 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-m6rpu3gh with job batch system ID: 21 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gyttywxo with job batch system ID: 22 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jvmeok8w with job batch system ID: 23 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xqryz_ub with job batch system ID: 24 and cores: 1, disk: 23.5 G, and memory: 6.0 G +[2020-11-27T18:51:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp28nxxy6y/worker_log.txt +[2020-11-27T18:51:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpiavg9wld/worker_log.txt +[2020-11-27T18:51:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdqc9gjkj/worker_log.txt +[2020-11-27T18:51:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpfj925aiz/worker_log.txt +[2020-11-27T18:51:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpca05603m/worker_log.txt +[2020-11-27T18:51:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpczpqtnjg/worker_log.txt +[2020-11-27T18:51:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpu3901gc9/worker_log.txt +[2020-11-27T18:51:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpzgzipkwy/worker_log.txt +[2020-11-27T18:51:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_9fgs680/worker_log.txt +[2020-11-27T18:51:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp2wpnrjk4/worker_log.txt +[2020-11-27T18:51:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1zvx1f76/worker_log.txt +[2020-11-27T18:51:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpf59m6m9r/worker_log.txt +[2020-11-27T18:51:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp6zrvdjm_/worker_log.txt +[2020-11-27T18:51:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_wtrktgy/worker_log.txt +[2020-11-27T18:51:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpubzbxoo4/worker_log.txt +[2020-11-27T18:51:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmprjepo8rp/worker_log.txt +[2020-11-27T18:51:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpp66ptrsm/worker_log.txt +[2020-11-27T18:51:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp383rm4r4/worker_log.txt +[2020-11-27T18:51:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpivwncu7m/worker_log.txt +[2020-11-27T18:51:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpjtrxl6bj/worker_log.txt +[2020-11-27T18:51:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp9spso3lu/worker_log.txt +[2020-11-27T18:51:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp4vluck__/worker_log.txt +[2020-11-27T18:51:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmplpjb8146/worker_log.txt +[2020-11-27T18:51:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpkzr8hpm6/worker_log.txt +[2020-11-27T18:51:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp0nip6cw0/worker_log.txt +[2020-11-27T18:51:42-0800] [MainThread] [I] [toil.leader] 25 jobs are running, 0 jobs are issued and waiting to run +[2020-11-27T19:27:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:16: unzipping dbsnp.vcf +[2020-11-27T19:27:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:16: unzipped dbsnp +[2020-11-27T19:27:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:16: configured input files for docker +[2020-11-27T19:27:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:16: starting mutect docker call +[2020-11-27T19:27:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:16: finshed docker, outputting now +[2020-11-27T19:27:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:16: Exporting files/for-job/kind-run_mutect_perchrom/instance-xqryz_ub/file-ibzpebdl/chrM.vcf to output location +[2020-11-27T19:27:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:16: Ran MuTect on TEST:chrM successfully +[2020-11-27T19:27:21-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-xqryz_ub +[2020-11-27T19:27:21-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ey28e5_5 with job batch system ID: 25 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:27:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp3is4qxl9/worker_log.txt +[2020-11-27T19:27:22-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ey28e5_5 +[2020-11-27T19:27:22-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-7kwh6teo with job batch system ID: 26 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:27:23-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpv25t9xrb/worker_log.txt +[2020-11-27T19:27:23-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-7kwh6teo +[2020-11-27T19:27:23-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:23: unzipping dbsnp.vcf +[2020-11-27T19:27:23-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:23: unzipped dbsnp +[2020-11-27T19:27:23-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:23: configured input files for docker +[2020-11-27T19:27:23-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:23: starting mutect docker call +[2020-11-27T19:27:23-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:23: finshed docker, outputting now +[2020-11-27T19:27:23-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:23: Exporting files/for-job/kind-run_mutect_perchrom/instance-mgq6ru10/file-byyos56q/chr19.vcf to output location +[2020-11-27T19:27:23-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:23: Ran MuTect on TEST:chr19 successfully +[2020-11-27T19:27:28-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-mgq6ru10 +[2020-11-27T19:27:28-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ngjd_q56 with job batch system ID: 27 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:27:29-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp720zskr5/worker_log.txt +[2020-11-27T19:27:29-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ngjd_q56 +[2020-11-27T19:27:29-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-ei8t4y9l with job batch system ID: 28 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:27:30-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpt1oqydkt/worker_log.txt +[2020-11-27T19:27:30-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-ei8t4y9l +[2020-11-27T19:27:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:56: unzipping dbsnp.vcf +[2020-11-27T19:27:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:56: unzipped dbsnp +[2020-11-27T19:27:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:56: configured input files for docker +[2020-11-27T19:27:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:56: starting mutect docker call +[2020-11-27T19:27:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:56: finshed docker, outputting now +[2020-11-27T19:27:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:56: Exporting files/for-job/kind-run_mutect_perchrom/instance-jvmeok8w/file-w659sqt7/chrY.vcf to output location +[2020-11-27T19:27:56-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:27:56: Ran MuTect on TEST:chrY successfully +[2020-11-27T19:28:01-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-jvmeok8w +[2020-11-27T19:28:01-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-vd702lg6 with job batch system ID: 29 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:28:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp42unfinz/worker_log.txt +[2020-11-27T19:28:02-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-vd702lg6 +[2020-11-27T19:28:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-f_o8awfa with job batch system ID: 30 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:28:03-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbtmvtynj/worker_log.txt +[2020-11-27T19:28:03-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-f_o8awfa +[2020-11-27T19:28:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:16: unzipping dbsnp.vcf +[2020-11-27T19:28:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:16: unzipped dbsnp +[2020-11-27T19:28:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:16: configured input files for docker +[2020-11-27T19:28:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:16: starting mutect docker call +[2020-11-27T19:28:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:16: finshed docker, outputting now +[2020-11-27T19:28:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:16: Exporting files/for-job/kind-run_mutect_perchrom/instance-m6rpu3gh/file-tsip0rtl/chr22.vcf to output location +[2020-11-27T19:28:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:16: Ran MuTect on TEST:chr22 successfully +[2020-11-27T19:28:22-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-m6rpu3gh +[2020-11-27T19:28:22-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-9ibizl8t with job batch system ID: 31 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:28:23-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpaf_mnd3m/worker_log.txt +[2020-11-27T19:28:23-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-9ibizl8t +[2020-11-27T19:28:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-030qvgzd with job batch system ID: 32 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:28:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpg3h1rk8d/worker_log.txt +[2020-11-27T19:28:24-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-030qvgzd +[2020-11-27T19:28:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:31: unzipping dbsnp.vcf +[2020-11-27T19:28:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:31: unzipped dbsnp +[2020-11-27T19:28:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:31: configured input files for docker +[2020-11-27T19:28:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:31: starting mutect docker call +[2020-11-27T19:28:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:31: finshed docker, outputting now +[2020-11-27T19:28:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:31: Exporting files/for-job/kind-run_mutect_perchrom/instance-ntmj9aw5/file-dhkkeyes/chr14.vcf to output location +[2020-11-27T19:28:31-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:31: Ran MuTect on TEST:chr14 successfully +[2020-11-27T19:28:36-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ntmj9aw5 +[2020-11-27T19:28:36-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-tym7l_46 with job batch system ID: 33 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:28:37-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1f9u5x7t/worker_log.txt +[2020-11-27T19:28:37-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-tym7l_46 +[2020-11-27T19:28:37-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-rnhvgo7a with job batch system ID: 34 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:28:38-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp316zos0j/worker_log.txt +[2020-11-27T19:28:38-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-rnhvgo7a +[2020-11-27T19:28:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:42: unzipping dbsnp.vcf +[2020-11-27T19:28:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:42: unzipped dbsnp +[2020-11-27T19:28:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:42: configured input files for docker +[2020-11-27T19:28:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:42: starting mutect docker call +[2020-11-27T19:28:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:42: finshed docker, outputting now +[2020-11-27T19:28:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:42: Exporting files/for-job/kind-run_mutect_perchrom/instance-77hux1xy/file-nnzqhdnp/chr21.vcf to output location +[2020-11-27T19:28:42-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:28:42: Ran MuTect on TEST:chr21 successfully +[2020-11-27T19:28:47-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-77hux1xy +[2020-11-27T19:28:47-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-9awvrwpx with job batch system ID: 35 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:28:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7m81qsv7/worker_log.txt +[2020-11-27T19:28:48-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-9awvrwpx +[2020-11-27T19:28:48-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-6xxfkfl2 with job batch system ID: 36 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:28:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpai5poed6/worker_log.txt +[2020-11-27T19:28:49-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-6xxfkfl2 +[2020-11-27T19:29:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:16: unzipping dbsnp.vcf +[2020-11-27T19:29:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:16: unzipped dbsnp +[2020-11-27T19:29:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:16: configured input files for docker +[2020-11-27T19:29:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:16: starting mutect docker call +[2020-11-27T19:29:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:16: finshed docker, outputting now +[2020-11-27T19:29:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:16: Exporting files/for-job/kind-run_mutect_perchrom/instance-p8mcfs96/file-ra0_epmo/chr20.vcf to output location +[2020-11-27T19:29:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:16: Ran MuTect on TEST:chr20 successfully +[2020-11-27T19:29:22-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-p8mcfs96 +[2020-11-27T19:29:22-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-fny9vudc with job batch system ID: 37 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:29:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpd6w1ka58/worker_log.txt +[2020-11-27T19:29:23-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-fny9vudc +[2020-11-27T19:29:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-7an9pegh with job batch system ID: 38 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:29:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp3bjelb3o/worker_log.txt +[2020-11-27T19:29:24-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-7an9pegh +[2020-11-27T19:29:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:26: unzipping dbsnp.vcf +[2020-11-27T19:29:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:26: unzipped dbsnp +[2020-11-27T19:29:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:26: configured input files for docker +[2020-11-27T19:29:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:26: starting mutect docker call +[2020-11-27T19:29:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:26: finshed docker, outputting now +[2020-11-27T19:29:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:26: Exporting files/for-job/kind-run_mutect_perchrom/instance-_u307ivu/file-hufvgnvh/chr16.vcf to output location +[2020-11-27T19:29:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:26: Ran MuTect on TEST:chr16 successfully +[2020-11-27T19:29:31-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-_u307ivu +[2020-11-27T19:29:31-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-2e_g8ivg with job batch system ID: 39 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:29:32-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmps8r98huo/worker_log.txt +[2020-11-27T19:29:32-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-2e_g8ivg +[2020-11-27T19:29:32-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-1zw81rwj with job batch system ID: 40 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:29:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpg5yi23ke/worker_log.txt +[2020-11-27T19:29:33-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-1zw81rwj +[2020-11-27T19:29:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:39: unzipping dbsnp.vcf +[2020-11-27T19:29:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:39: unzipped dbsnp +[2020-11-27T19:29:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:39: configured input files for docker +[2020-11-27T19:29:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:39: starting mutect docker call +[2020-11-27T19:29:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:39: finshed docker, outputting now +[2020-11-27T19:29:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:39: Exporting files/for-job/kind-run_mutect_perchrom/instance-hxsjhjvp/file-1uqaomm1/chr13.vcf to output location +[2020-11-27T19:29:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:39: Ran MuTect on TEST:chr13 successfully +[2020-11-27T19:29:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:40: unzipping dbsnp.vcf +[2020-11-27T19:29:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:40: unzipped dbsnp +[2020-11-27T19:29:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:40: configured input files for docker +[2020-11-27T19:29:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:40: starting mutect docker call +[2020-11-27T19:29:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:40: finshed docker, outputting now +[2020-11-27T19:29:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:40: Exporting files/for-job/kind-run_mutect_perchrom/instance-du24kqrt/file-er_w3mmm/chr17.vcf to output location +[2020-11-27T19:29:40-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:40: Ran MuTect on TEST:chr17 successfully +[2020-11-27T19:29:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-hxsjhjvp +[2020-11-27T19:29:44-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-w2kp_jod with job batch system ID: 41 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:29:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpr_6v06is/worker_log.txt +[2020-11-27T19:29:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-du24kqrt +[2020-11-27T19:29:45-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-vb50qcqs with job batch system ID: 42 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:29:45-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-w2kp_jod +[2020-11-27T19:29:45-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-z1dbqpwz with job batch system ID: 43 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:29:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpfppn_eci/worker_log.txt +[2020-11-27T19:29:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpb4v53j44/worker_log.txt +[2020-11-27T19:29:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-z1dbqpwz +[2020-11-27T19:29:46-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-vb50qcqs +[2020-11-27T19:29:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-t670_m75 with job batch system ID: 44 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:29:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpi01zgyvu/worker_log.txt +[2020-11-27T19:29:47-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-t670_m75 +[2020-11-27T19:29:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:54: unzipping dbsnp.vcf +[2020-11-27T19:29:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:54: unzipped dbsnp +[2020-11-27T19:29:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:54: configured input files for docker +[2020-11-27T19:29:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:54: starting mutect docker call +[2020-11-27T19:29:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:54: finshed docker, outputting now +[2020-11-27T19:29:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:54: Exporting files/for-job/kind-run_mutect_perchrom/instance-5i9__ll1/file-rwwigl8u/chr18.vcf to output location +[2020-11-27T19:29:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:29:54: Ran MuTect on TEST:chr18 successfully +[2020-11-27T19:29:59-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5i9__ll1 +[2020-11-27T19:29:59-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-axwxldsm with job batch system ID: 45 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:30:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpiyxsa33w/worker_log.txt +[2020-11-27T19:30:00-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-axwxldsm +[2020-11-27T19:30:00-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-rlk8veoh with job batch system ID: 46 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:30:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpw1spmmw0/worker_log.txt +[2020-11-27T19:30:01-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-rlk8veoh +[2020-11-27T19:30:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:03: unzipping dbsnp.vcf +[2020-11-27T19:30:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:03: unzipped dbsnp +[2020-11-27T19:30:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:03: configured input files for docker +[2020-11-27T19:30:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:03: starting mutect docker call +[2020-11-27T19:30:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:03: finshed docker, outputting now +[2020-11-27T19:30:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:03: Exporting files/for-job/kind-run_mutect_perchrom/instance-e1yejtsm/file-phyh8s8u/chr15.vcf to output location +[2020-11-27T19:30:03-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:03: Ran MuTect on TEST:chr15 successfully +[2020-11-27T19:30:08-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-e1yejtsm +[2020-11-27T19:30:08-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-agnrge02 with job batch system ID: 47 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:30:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpiibarirt/worker_log.txt +[2020-11-27T19:30:09-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-agnrge02 +[2020-11-27T19:30:09-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-pihl7nio with job batch system ID: 48 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:30:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpp0pjvzhf/worker_log.txt +[2020-11-27T19:30:10-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-pihl7nio +[2020-11-27T19:30:12-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:12: unzipping dbsnp.vcf +[2020-11-27T19:30:12-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:12: unzipped dbsnp +[2020-11-27T19:30:12-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:12: configured input files for docker +[2020-11-27T19:30:12-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:12: starting mutect docker call +[2020-11-27T19:30:12-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:12: finshed docker, outputting now +[2020-11-27T19:30:12-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:12: Exporting files/for-job/kind-run_mutect_perchrom/instance-gyttywxo/file-i7hkgwmb/chrX.vcf to output location +[2020-11-27T19:30:12-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:12: Ran MuTect on TEST:chrX successfully +[2020-11-27T19:30:18-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-gyttywxo +[2020-11-27T19:30:18-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-qatscu9_ with job batch system ID: 49 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:30:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmps6lc1re5/worker_log.txt +[2020-11-27T19:30:19-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-qatscu9_ +[2020-11-27T19:30:19-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-_90_leu0 with job batch system ID: 50 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:30:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp795382i1/worker_log.txt +[2020-11-27T19:30:20-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-_90_leu0 +[2020-11-27T19:30:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:55: unzipping dbsnp.vcf +[2020-11-27T19:30:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:55: unzipped dbsnp +[2020-11-27T19:30:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:55: configured input files for docker +[2020-11-27T19:30:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:55: starting mutect docker call +[2020-11-27T19:30:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:55: finshed docker, outputting now +[2020-11-27T19:30:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:55: Exporting files/for-job/kind-run_mutect_perchrom/instance-f3zgk1bv/file-5yijhene/chr9.vcf to output location +[2020-11-27T19:30:55-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:30:55: Ran MuTect on TEST:chr9 successfully +[2020-11-27T19:31:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-f3zgk1bv +[2020-11-27T19:31:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-a89bt18q with job batch system ID: 51 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:31:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7dnedzjs/worker_log.txt +[2020-11-27T19:31:02-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-a89bt18q +[2020-11-27T19:31:02-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-kc69f8vr with job batch system ID: 52 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:31:02-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:02: unzipping dbsnp.vcf +[2020-11-27T19:31:02-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:02: unzipped dbsnp +[2020-11-27T19:31:02-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:02: configured input files for docker +[2020-11-27T19:31:02-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:02: starting mutect docker call +[2020-11-27T19:31:02-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:02: finshed docker, outputting now +[2020-11-27T19:31:02-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:02: Exporting files/for-job/kind-run_mutect_perchrom/instance-9i484bqi/file-mmren2xk/chr11.vcf to output location +[2020-11-27T19:31:02-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:02: Ran MuTect on TEST:chr11 successfully +[2020-11-27T19:31:02-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpebufkps8/worker_log.txt +[2020-11-27T19:31:02-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-kc69f8vr +[2020-11-27T19:31:07-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-9i484bqi +[2020-11-27T19:31:07-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-i1pusei2 with job batch system ID: 53 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:31:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpn7ox2039/worker_log.txt +[2020-11-27T19:31:08-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-i1pusei2 +[2020-11-27T19:31:08-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-e3gki4ix with job batch system ID: 54 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:31:09-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp36a7bz8i/worker_log.txt +[2020-11-27T19:31:09-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-e3gki4ix +[2020-11-27T19:31:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:14: unzipping dbsnp.vcf +[2020-11-27T19:31:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:14: unzipped dbsnp +[2020-11-27T19:31:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:14: configured input files for docker +[2020-11-27T19:31:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:14: starting mutect docker call +[2020-11-27T19:31:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:14: finshed docker, outputting now +[2020-11-27T19:31:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:14: Exporting files/for-job/kind-run_mutect_perchrom/instance-h0tdg49o/file-f3w9txq2/chr12.vcf to output location +[2020-11-27T19:31:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:14: Ran MuTect on TEST:chr12 successfully +[2020-11-27T19:31:19-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-h0tdg49o +[2020-11-27T19:31:19-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-63vxbniz with job batch system ID: 55 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:31:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:20: unzipping dbsnp.vcf +[2020-11-27T19:31:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:20: unzipped dbsnp +[2020-11-27T19:31:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:20: configured input files for docker +[2020-11-27T19:31:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:20: starting mutect docker call +[2020-11-27T19:31:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:20: finshed docker, outputting now +[2020-11-27T19:31:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:20: Exporting files/for-job/kind-run_mutect_perchrom/instance-r0a3rxak/file-j_cp69xe/chr10.vcf to output location +[2020-11-27T19:31:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:20: Ran MuTect on TEST:chr10 successfully +[2020-11-27T19:31:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpe7g474de/worker_log.txt +[2020-11-27T19:31:20-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-63vxbniz +[2020-11-27T19:31:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-l0norp0m with job batch system ID: 56 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:31:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpyswmr7sy/worker_log.txt +[2020-11-27T19:31:21-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-l0norp0m +[2020-11-27T19:31:25-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-r0a3rxak +[2020-11-27T19:31:25-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-at6zynqq with job batch system ID: 57 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:31:25-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp5gkt1fnm/worker_log.txt +[2020-11-27T19:31:26-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-at6zynqq +[2020-11-27T19:31:26-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-l7vpzyz4 with job batch system ID: 58 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:31:26-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpel09vqd0/worker_log.txt +[2020-11-27T19:31:26-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-l7vpzyz4 +[2020-11-27T19:31:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:28: unzipping dbsnp.vcf +[2020-11-27T19:31:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:28: unzipped dbsnp +[2020-11-27T19:31:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:28: configured input files for docker +[2020-11-27T19:31:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:28: starting mutect docker call +[2020-11-27T19:31:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:28: finshed docker, outputting now +[2020-11-27T19:31:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:28: Exporting files/for-job/kind-run_mutect_perchrom/instance-y2fcywgt/file-z6e4g9ga/chr8.vcf to output location +[2020-11-27T19:31:28-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:31:28: Ran MuTect on TEST:chr8 successfully +[2020-11-27T19:31:33-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-y2fcywgt +[2020-11-27T19:31:33-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-658__1nb with job batch system ID: 59 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:31:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpmf63j0pc/worker_log.txt +[2020-11-27T19:31:34-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-658__1nb +[2020-11-27T19:31:34-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-wlvnhdem with job batch system ID: 60 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:31:35-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpwph7238p/worker_log.txt +[2020-11-27T19:31:35-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-wlvnhdem +[2020-11-27T19:32:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:01: unzipping dbsnp.vcf +[2020-11-27T19:32:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:01: unzipped dbsnp +[2020-11-27T19:32:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:01: configured input files for docker +[2020-11-27T19:32:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:01: starting mutect docker call +[2020-11-27T19:32:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:01: finshed docker, outputting now +[2020-11-27T19:32:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:01: Exporting files/for-job/kind-run_mutect_perchrom/instance-4f2ha64d/file-72wquezo/chr6.vcf to output location +[2020-11-27T19:32:01-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:01: Ran MuTect on TEST:chr6 successfully +[2020-11-27T19:32:06-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-4f2ha64d +[2020-11-27T19:32:06-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-kchgky60 with job batch system ID: 61 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:32:07-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpnv4zmjz5/worker_log.txt +[2020-11-27T19:32:07-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-kchgky60 +[2020-11-27T19:32:07-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-yjn16wtm with job batch system ID: 62 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:32:08-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp716usih0/worker_log.txt +[2020-11-27T19:32:08-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-yjn16wtm +[2020-11-27T19:32:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:26: unzipping dbsnp.vcf +[2020-11-27T19:32:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:26: unzipped dbsnp +[2020-11-27T19:32:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:26: configured input files for docker +[2020-11-27T19:32:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:26: starting mutect docker call +[2020-11-27T19:32:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:26: finshed docker, outputting now +[2020-11-27T19:32:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:26: Exporting files/for-job/kind-run_mutect_perchrom/instance-1jh6nqug/file-eurb2w7v/chr7.vcf to output location +[2020-11-27T19:32:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:26: Ran MuTect on TEST:chr7 successfully +[2020-11-27T19:32:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:26: unzipping dbsnp.vcf +[2020-11-27T19:32:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:26: unzipped dbsnp +[2020-11-27T19:32:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:26: configured input files for docker +[2020-11-27T19:32:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:26: starting mutect docker call +[2020-11-27T19:32:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:26: finshed docker, outputting now +[2020-11-27T19:32:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:26: Exporting files/for-job/kind-run_mutect_perchrom/instance-ey78_apt/file-1femcant/chr5.vcf to output location +[2020-11-27T19:32:26-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:26: Ran MuTect on TEST:chr5 successfully +[2020-11-27T19:32:32-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-ey78_apt +[2020-11-27T19:32:32-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-r52wwzhy with job batch system ID: 63 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:32:32-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-1jh6nqug +[2020-11-27T19:32:32-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-vn_5q21z with job batch system ID: 64 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:32:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpe_wvgt0n/worker_log.txt +[2020-11-27T19:32:33-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-r52wwzhy +[2020-11-27T19:32:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-29o5kgl4 with job batch system ID: 65 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:32:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpw9gxxnad/worker_log.txt +[2020-11-27T19:32:33-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-vn_5q21z +[2020-11-27T19:32:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-dz5pfkoc with job batch system ID: 66 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:32:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmporie0h8c/worker_log.txt +[2020-11-27T19:32:34-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-29o5kgl4 +[2020-11-27T19:32:34-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp64s7hs7q/worker_log.txt +[2020-11-27T19:32:34-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-dz5pfkoc +[2020-11-27T19:32:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:54: unzipping dbsnp.vcf +[2020-11-27T19:32:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:54: unzipped dbsnp +[2020-11-27T19:32:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:54: configured input files for docker +[2020-11-27T19:32:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:54: starting mutect docker call +[2020-11-27T19:32:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:54: finshed docker, outputting now +[2020-11-27T19:32:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:54: Exporting files/for-job/kind-run_mutect_perchrom/instance-fgj0w2_w/file-ehwa66xg/chr4.vcf to output location +[2020-11-27T19:32:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:32:54: Ran MuTect on TEST:chr4 successfully +[2020-11-27T19:33:00-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-fgj0w2_w +[2020-11-27T19:33:00-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-fc9l_3e0 with job batch system ID: 67 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:33:00-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp54db86c1/worker_log.txt +[2020-11-27T19:33:00-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-fc9l_3e0 +[2020-11-27T19:33:00-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-f7d6aq5_ with job batch system ID: 68 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:33:01-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpq_6smjg1/worker_log.txt +[2020-11-27T19:33:01-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-f7d6aq5_ +[2020-11-27T19:33:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:33:14: unzipping dbsnp.vcf +[2020-11-27T19:33:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:33:14: unzipped dbsnp +[2020-11-27T19:33:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:33:14: configured input files for docker +[2020-11-27T19:33:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:33:14: starting mutect docker call +[2020-11-27T19:33:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:33:14: finshed docker, outputting now +[2020-11-27T19:33:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:33:14: Exporting files/for-job/kind-run_mutect_perchrom/instance-989tgher/file-6o4kevln/chr3.vcf to output location +[2020-11-27T19:33:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:33:14: Ran MuTect on TEST:chr3 successfully +[2020-11-27T19:33:20-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-989tgher +[2020-11-27T19:33:20-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-fpnlf0vq with job batch system ID: 69 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:33:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_genjbll/worker_log.txt +[2020-11-27T19:33:21-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-fpnlf0vq +[2020-11-27T19:33:21-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-npf1btg7 with job batch system ID: 70 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:33:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpcp8988if/worker_log.txt +[2020-11-27T19:33:22-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-npf1btg7 +[2020-11-27T19:33:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:33:37: unzipping dbsnp.vcf +[2020-11-27T19:33:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:33:37: unzipped dbsnp +[2020-11-27T19:33:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:33:37: configured input files for docker +[2020-11-27T19:33:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:33:37: starting mutect docker call +[2020-11-27T19:33:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:33:37: finshed docker, outputting now +[2020-11-27T19:33:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:33:37: Exporting files/for-job/kind-run_mutect_perchrom/instance-t15qgjy0/file-nx3mii86/chr2.vcf to output location +[2020-11-27T19:33:37-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:33:37: Ran MuTect on TEST:chr2 successfully +[2020-11-27T19:33:42-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-t15qgjy0 +[2020-11-27T19:33:42-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-i03h6y20 with job batch system ID: 71 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:33:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp9zu_k2m3/worker_log.txt +[2020-11-27T19:33:43-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-i03h6y20 +[2020-11-27T19:33:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-dkq5pakk with job batch system ID: 72 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:33:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpa9l20unk/worker_log.txt +[2020-11-27T19:33:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-dkq5pakk +[2020-11-27T19:34:06-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:06: unzipping dbsnp.vcf +[2020-11-27T19:34:06-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:06: unzipped dbsnp +[2020-11-27T19:34:06-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:06: configured input files for docker +[2020-11-27T19:34:06-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:06: starting mutect docker call +[2020-11-27T19:34:06-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:06: finshed docker, outputting now +[2020-11-27T19:34:06-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:06: Exporting files/for-job/kind-run_mutect_perchrom/instance-5zdtbtsn/file-0zghnu93/chr1.vcf to output location +[2020-11-27T19:34:06-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:06: Ran MuTect on TEST:chr1 successfully +[2020-11-27T19:34:11-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-run_mutect_perchrom/instance-5zdtbtsn +[2020-11-27T19:34:11-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-x4ugfdbr with job batch system ID: 73 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:34:12-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpfc6lhycg/worker_log.txt +[2020-11-27T19:34:12-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-x4ugfdbr +[2020-11-27T19:34:12-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect_perchrom' kind-EncapsulatedJob/instance-yzx302yq with job batch system ID: 74 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:34:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1vlyx18h/worker_log.txt +[2020-11-27T19:34:13-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect_perchrom' kind-EncapsulatedJob/instance-yzx302yq +[2020-11-27T19:34:13-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-rong8s8v with job batch system ID: 75 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:34:13-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp5mmuguxe/worker_log.txt +[2020-11-27T19:34:13-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-rong8s8v +[2020-11-27T19:34:13-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-_w1f8gtw with job batch system ID: 76 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T19:34:14-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7l5r1yh9/worker_log.txt +[2020-11-27T19:34:14-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:14: Aggregated mutations for TEST successfully +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-_w1f8gtw +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-9z773tpr with job batch system ID: 77 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-ft1vefev with job batch system ID: 78 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-836_00fn with job batch system ID: 79 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-redg2qyx with job batch system ID: 80 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-5_e3d94i with job batch system ID: 81 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-aqtytsn8 with job batch system ID: 82 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-t9vtdihl with job batch system ID: 83 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-otj75j2y with job batch system ID: 84 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-u6wht2_v with job batch system ID: 85 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-qzodfsg_ with job batch system ID: 86 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-yxx95nf4 with job batch system ID: 87 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-bch7cwh9 with job batch system ID: 88 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-dyxkfwq2 with job batch system ID: 89 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-arlwozsc with job batch system ID: 90 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-qmg9q837 with job batch system ID: 91 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-l2k7ypdh with job batch system ID: 92 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-r0kx4zjt with job batch system ID: 93 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-tfwjbyc3 with job batch system ID: 94 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-pani4kva with job batch system ID: 95 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-n__5cy5h with job batch system ID: 96 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-k6oyqide with job batch system ID: 97 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-24tmw_7t with job batch system ID: 98 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-qsa0yy75 with job batch system ID: 99 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-sfgrzuxv with job batch system ID: 100 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:15-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-94592i89 with job batch system ID: 101 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpq8a88ta7/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp5qp9pft_/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpmvadhsvx/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbeq7gf48/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp6czrdzot/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1i4vaxyy/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmplshr0gin/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdhmg8n7q/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdy1r1ry9/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp0sn3_9dm/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp0mi4a7cr/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpfeipfs2s/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpnv41a6al/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7rsd7cs7/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpynokbhf2/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpqec9ue8z/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpy82ddbmz/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpxuzcr2gh/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmphwxe106s/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_fiz34tx/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpvkhpitxz/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpzviqetw0/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpc9plzwr_/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbt6e10_z/worker_log.txt +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmppp3ofszk/worker_log.txt +[2020-11-27T19:34:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:16: process mutect vcf start +[2020-11-27T19:34:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:16: Exporting files/for-job/kind-merge_perchrom_mutations/instance-9z773tpr/file-p9qlgvol/chr1.vcf to output location +[2020-11-27T19:34:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:16: process mutect vcf start +[2020-11-27T19:34:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:16: Exporting files/for-job/kind-merge_perchrom_mutations/instance-aqtytsn8/file-hrqklguu/chr6.vcf to output location +[2020-11-27T19:34:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:16: process mutect vcf start +[2020-11-27T19:34:16-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:16: Exporting files/for-job/kind-merge_perchrom_mutations/instance-otj75j2y/file-i0d78rho/chr8.vcf to output location +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-aqtytsn8 +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-9z773tpr +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-otj75j2y +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-bch7cwh9 +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-l2k7ypdh +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-redg2qyx +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-qzodfsg_ +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-n__5cy5h +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-5_e3d94i +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-qmg9q837 +[2020-11-27T19:34:16-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-t9vtdihl +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-bch7cwh9/file-1914ngjq/chr12.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-redg2qyx/file-_jp8yyzn/chr4.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-l2k7ypdh/file-3h0aixcm/chr16.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-qzodfsg_/file-0bhft75i/chr10.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-n__5cy5h/file-98f_59ki/chr20.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-5_e3d94i/file-p0_ehd2z/chr5.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-qmg9q837/file-tnq47yuz/chr15.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-t9vtdihl/file-c7fabua6/chr7.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-k6oyqide/file-pkah_qne/chr21.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-yxx95nf4/file-5odtfody/chr11.vcf to output location +[2020-11-27T19:34:17-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-k6oyqide +[2020-11-27T19:34:17-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-yxx95nf4 +[2020-11-27T19:34:17-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-u6wht2_v +[2020-11-27T19:34:17-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-qsa0yy75 +[2020-11-27T19:34:17-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-sfgrzuxv +[2020-11-27T19:34:17-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-arlwozsc +[2020-11-27T19:34:17-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-tfwjbyc3 +[2020-11-27T19:34:17-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-94592i89 +[2020-11-27T19:34:17-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-836_00fn +[2020-11-27T19:34:17-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-dyxkfwq2 +[2020-11-27T19:34:17-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-r0kx4zjt +[2020-11-27T19:34:17-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-ft1vefev +[2020-11-27T19:34:17-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-pani4kva +[2020-11-27T19:34:17-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_mutations' kind-merge_perchrom_mutations/instance-24tmw_7t +[2020-11-27T19:34:17-0800] [MainThread] [I] [toil.leader] Issued job 'merge_perchrom_vcfs' kind-merge_perchrom_vcfs/instance-h1k9mt2n with job batch system ID: 102 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-arlwozsc/file-wog9abso/chr14.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-u6wht2_v/file-3k7rhiwl/chr9.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-qsa0yy75/file-8csy2621/chrX.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-sfgrzuxv/file-ptvtfaox/chrY.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-94592i89/file-wuqvxzlv/chrM.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-tfwjbyc3/file-nli27x59/chr18.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-pani4kva/file-ftt2_iwp/chr19.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-r0kx4zjt/file-3lzwhfp1/chr17.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-dyxkfwq2/file-a1h_neqc/chr13.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-836_00fn/file-2hn3mau7/chr3.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-24tmw_7t/file-u50i9gfb/chr22.vcf to output location +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: process mutect vcf start +[2020-11-27T19:34:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:17: Exporting files/for-job/kind-merge_perchrom_mutations/instance-ft1vefev/file-yyf4_us3/chr2.vcf to output location +[2020-11-27T19:34:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpimnqzm85/worker_log.txt +[2020-11-27T19:34:18-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_perchrom_vcfs' kind-merge_perchrom_vcfs/instance-h1k9mt2n +[2020-11-27T19:34:18-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-bn2q7ksu with job batch system ID: 103 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:34:18-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:18: Exporting files/for-job/kind-merge_perchrom_vcfs/instance-h1k9mt2n/file-rw554xsd/all_merged.vcf to output location +[2020-11-27T19:34:18-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:18: Ran merge_perchrom_vcfs for merged successfully +[2020-11-27T19:34:18-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpnm45sm6f/worker_log.txt +[2020-11-27T19:34:19-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-bn2q7ksu +[2020-11-27T19:34:19-0800] [MainThread] [I] [toil.leader] Issued job 'delete_bams' kind-delete_bams/instance-ia_03zca with job batch system ID: 104 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T19:34:19-0800] [MainThread] [I] [toil.leader] Issued job 'delete_bams' kind-delete_bams/instance-yr_262lz with job batch system ID: 105 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T19:34:19-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-1rgbxdn2 with job batch system ID: 106 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:34:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpqsjmylsc/worker_log.txt +[2020-11-27T19:34:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpyfygnuva/worker_log.txt +[2020-11-27T19:34:19-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpop6l4_4b/worker_log.txt +[2020-11-27T19:34:20-0800] [MainThread] [I] [toil.leader] Job ended: 'delete_bams' kind-delete_bams/instance-ia_03zca +[2020-11-27T19:34:20-0800] [MainThread] [I] [toil.leader] Job ended: 'delete_bams' kind-delete_bams/instance-yr_262lz +[2020-11-27T19:34:20-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-1rgbxdn2 +[2020-11-27T19:34:20-0800] [MainThread] [I] [toil.leader] Issued job 'run_snpeff' kind-run_snpeff/instance-wm1d36r1 with job batch system ID: 107 and cores: 1, disk: 6.0 G, and memory: 2.0 G +[2020-11-27T19:34:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:20: Deleting "tumor_dna_fix_pg_sorted.bam" for patient "TEST". +[2020-11-27T19:34:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:20: Deleting "tumor_dna_fix_pg_sorted.bam.bai" for patient "TEST". +[2020-11-27T19:34:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:20: Deleting "normal_dna_fix_pg_sorted.bam" for patient "TEST". +[2020-11-27T19:34:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:34:20: Deleting "normal_dna_fix_pg_sorted.bam.bai" for patient "TEST". +[2020-11-27T19:34:20-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpwcz2cuoc/worker_log.txt +[2020-11-27T19:36:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:36:20: Exporting files/for-job/kind-run_snpeff/instance-wm1d36r1/file-8zf4wy79/mutations.vcf to output location +[2020-11-27T19:36:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 19:36:20: Ran snpeff on TEST successfully +[2020-11-27T19:36:21-0800] [MainThread] [I] [toil.leader] Job ended: 'run_snpeff' kind-run_snpeff/instance-wm1d36r1 +[2020-11-27T19:36:21-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-770lc4lu with job batch system ID: 108 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T19:36:22-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpmigm34pd/worker_log.txt +[2020-11-27T19:36:22-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-770lc4lu +[2020-11-27T19:36:22-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-vevukocf with job batch system ID: 109 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T19:36:23-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpqwbyafk3/worker_log.txt +[2020-11-27T19:36:23-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-vevukocf +[2020-11-27T19:36:23-0800] [MainThread] [I] [toil.leader] Issued job 'run_transgene' kind-run_transgene/instance-qj3j385j with job batch system ID: 110 and cores: 1, disk: 310.2 M, and memory: 100.0 M +[2020-11-27T19:36:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpovwwmtjn/worker_log.txt +[2020-11-27T19:37:39-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_transgene file:/scratch/drkthomp/jobStore kind-run_transgene/instance-qj3j385j. +[2020-11-27T19:37:39-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_transgene' kind-run_transgene/instance-qj3j385j +[2020-11-27T19:37:39-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_transgene' kind-run_transgene/instance-qj3j385j +[2020-11-27T19:37:39-0800] [MainThread] [W] [toil.leader] Log from job kind-run_transgene/instance-qj3j385j follows: +=========> + [2020-11-27T19:36:24-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T19:36:24-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + 2.5.0: Pulling from aarjunrao/transgene + 04c460fac791: Already exists + 0a0916b29f3e: Already exists + b25f4e7a7766: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + 595fd1429200: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + 95b1864271d3: Already exists + 08455334bb58: Already exists + d79e1b284af7: Already exists + 6022965ff030: Already exists + a3ed95caeb02: Already exists + 5397d43b7e0a: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + 08d436ec9dcd: Already exists + 1681d1d31560: Already exists + a056f67462ec: Already exists + 3e6cd62720d9: Already exists + a3ed95caeb02: Already exists + 6ccb234d8efe: Already exists + a3ed95caeb02: Already exists + d3e04f19bd90: Pulling fs layer + 120cdff75942: Pulling fs layer + 93815a1b4f2c: Pulling fs layer + 8ec7ea6e5e66: Pulling fs layer + a3ed95caeb02: Pulling fs layer + a3ed95caeb02: Waiting + 8ec7ea6e5e66: Waiting + 93815a1b4f2c: Verifying Checksum + 93815a1b4f2c: Download complete + 120cdff75942: Verifying Checksum + 120cdff75942: Download complete + 8ec7ea6e5e66: Verifying Checksum + 8ec7ea6e5e66: Download complete + a3ed95caeb02: Download complete + d3e04f19bd90: Verifying Checksum + d3e04f19bd90: Download complete + d3e04f19bd90: Pull complete + 120cdff75942: Pull complete + 93815a1b4f2c: Pull complete + 8ec7ea6e5e66: Pull complete + a3ed95caeb02: Pull complete + Digest: sha256:6f04cc98211a4d1fa46a54cf43d8a6de25e69ba715485ab037fbda6610090d00 + Status: Downloaded newer image for aarjunrao/transgene:2.5.0 + usage: transgene [-h] [--peptides PEPTIDE_FILE] + [--snpeff SNPEFF_FILE | --vep VEP_FILE] + [--genome GENOME_FILE] [--indel_extend_length EXTEND_LENGTH] + [--fusions FUSION_FILE] [--transcripts TRANSCRIPT_FILE] + [--annotation ANNOTATION_FILE] [--filter_mt_fusions] + [--filter_ig_pairs] [--filter_rna_gene_fusions] + [--filter_readthroughs] + [--readthrough_threshold RT_THRESHOLD] [--rna_file RNA_FILE] + [--reject_threshold REJECT_THRESHOLD] + [--min_rna_alt_freq RNA_MIN_ALT_FREQ] [--filterOxoG] + [--dna_file DNA_FILE] + [--min_OxoG_variant_freq OXOG_MIN_ALT_FREQ] + [--log_level {INFO,DEBUG,WARNING,ERROR}] [--log_file LOGFILE] + --prefix PREFIX [--pep_lens PEP_LENS] [--no_json_dumps] + [--cores CORES] + transgene: error: argument --cores: invalid int value: '80.0' + [2020-11-27T19:37:38-0800] [MainThread] [W] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-vevukocf/cleanup/file-rasmyjg9/stream used 1817.94% (5.5 GB [5914071040B] used, 310.2 MB [325317817B] requested) at the end of its run. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 137, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpovwwmtjn/95d14368-7407-43e7-baa0-8c5c7ce5f40f:/data', '--log-driver=none', 'aarjunrao/transgene:2.5.0', '--peptides', '/data/gencode.v25.pc_translations_NOPARY.fa', '--prefix', 'transgened', '--pep_lens', '9,10,15', '--cores', '80.0', '--genome', '/data/hg38.fa', '--annotation', '/data/gencode.v25.annotation_NOPARY.gtf', '--snpeff', '/data/snpeffed_muts.vcf', '--rna_file', '/data/rna.bam', '--transcripts', '/data/gencode.v25.pc_transcripts_NOPARY.fa', '--fusions', '/data/fusion_calls']' returned non-zero exit status 2. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_translation.py", line 118, in run_transgene + tool_version=transgene_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 140, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (2)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpovwwmtjn/95d14368-7407-43e7-baa0-8c5c7ce5f40f:/data --log-driver=none aarjunrao/transgene:2.5.0 --peptides /data/gencode.v25.pc_translations_NOPARY.fa --prefix transgened --pep_lens 9,10,15 --cores 80.0 --genome /data/hg38.fa --annotation /data/gencode.v25.annotation_NOPARY.gtf --snpeff /data/snpeffed_muts.vcf --rna_file /data/rna.bam --transcripts /data/gencode.v25.pc_transcripts_NOPARY.fa --fusions /data/fusion_calls" + [2020-11-27T19:37:38-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-27T19:37:39-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_transgene' kind-run_transgene/instance-qj3j385j with ID kind-run_transgene/instance-qj3j385j to 1 +[2020-11-27T19:37:39-0800] [MainThread] [W] [toil.jobGraph] We have increased the default memory of the failed job 'run_transgene' kind-run_transgene/instance-qj3j385j to 2147483648 bytes +[2020-11-27T19:37:39-0800] [MainThread] [W] [toil.jobGraph] We have increased the disk of the failed job 'run_transgene' kind-run_transgene/instance-qj3j385j to the default of 2147483648 bytes +[2020-11-27T19:37:39-0800] [MainThread] [I] [toil.leader] Issued job 'run_transgene' kind-run_transgene/instance-qj3j385j with job batch system ID: 111 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T19:37:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpeek2yuwe/worker_log.txt +[2020-11-27T19:38:47-0800] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker run_transgene file:/scratch/drkthomp/jobStore kind-run_transgene/instance-qj3j385j. +[2020-11-27T19:38:47-0800] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_transgene' kind-run_transgene/instance-qj3j385j +[2020-11-27T19:38:47-0800] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_transgene' kind-run_transgene/instance-qj3j385j +[2020-11-27T19:38:47-0800] [MainThread] [W] [toil.leader] Log from job kind-run_transgene/instance-qj3j385j follows: +=========> + [2020-11-27T19:37:40-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T19:37:40-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + usage: transgene [-h] [--peptides PEPTIDE_FILE] + [--snpeff SNPEFF_FILE | --vep VEP_FILE] + [--genome GENOME_FILE] [--indel_extend_length EXTEND_LENGTH] + [--fusions FUSION_FILE] [--transcripts TRANSCRIPT_FILE] + [--annotation ANNOTATION_FILE] [--filter_mt_fusions] + [--filter_ig_pairs] [--filter_rna_gene_fusions] + [--filter_readthroughs] + [--readthrough_threshold RT_THRESHOLD] [--rna_file RNA_FILE] + [--reject_threshold REJECT_THRESHOLD] + [--min_rna_alt_freq RNA_MIN_ALT_FREQ] [--filterOxoG] + [--dna_file DNA_FILE] + [--min_OxoG_variant_freq OXOG_MIN_ALT_FREQ] + [--log_level {INFO,DEBUG,WARNING,ERROR}] [--log_file LOGFILE] + --prefix PREFIX [--pep_lens PEP_LENS] [--no_json_dumps] + [--cores CORES] + transgene: error: argument --cores: invalid int value: '80.0' + [2020-11-27T19:38:46-0800] [MainThread] [W] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-vevukocf/cleanup/file-rasmyjg9/stream used 1817.94% (5.5 GB [5914071040B] used, 310.2 MB [325317817B] requested) at the end of its run. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 137, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpeek2yuwe/a6ab82f8-4fb2-47c7-aca6-8dc94f9073ff:/data', '--log-driver=none', 'aarjunrao/transgene:2.5.0', '--peptides', '/data/gencode.v25.pc_translations_NOPARY.fa', '--prefix', 'transgened', '--pep_lens', '9,10,15', '--cores', '80.0', '--genome', '/data/hg38.fa', '--annotation', '/data/gencode.v25.annotation_NOPARY.gtf', '--snpeff', '/data/snpeffed_muts.vcf', '--rna_file', '/data/rna.bam', '--transcripts', '/data/gencode.v25.pc_transcripts_NOPARY.fa', '--fusions', '/data/fusion_calls']' returned non-zero exit status 2. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_translation.py", line 118, in run_transgene + tool_version=transgene_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 140, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (2)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpeek2yuwe/a6ab82f8-4fb2-47c7-aca6-8dc94f9073ff:/data --log-driver=none aarjunrao/transgene:2.5.0 --peptides /data/gencode.v25.pc_translations_NOPARY.fa --prefix transgened --pep_lens 9,10,15 --cores 80.0 --genome /data/hg38.fa --annotation /data/gencode.v25.annotation_NOPARY.gtf --snpeff /data/snpeffed_muts.vcf --rna_file /data/rna.bam --transcripts /data/gencode.v25.pc_transcripts_NOPARY.fa --fusions /data/fusion_calls" + [2020-11-27T19:38:46-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= +[2020-11-27T19:38:47-0800] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_transgene' kind-run_transgene/instance-qj3j385j with ID kind-run_transgene/instance-qj3j385j to 0 +[2020-11-27T19:38:47-0800] [MainThread] [W] [toil.leader] Job 'run_transgene' kind-run_transgene/instance-qj3j385j with ID kind-run_transgene/instance-qj3j385j is completely failed +[2020-11-27T19:39:22-0800] [MainThread] [I] [toil.leader] Finished toil run with 42 failed jobs. +[2020-11-27T19:39:22-0800] [MainThread] [I] [toil.leader] Failed jobs at end of the run: 'run_muse' kind-EncapsulatedJob/instance-qi2fqyz_ 'run_transgene' kind-EncapsulatedJob/instance-vevukocf 'Job' kind-Job/instance-7s1jyb6z 'Job' kind-Job/instance-wizon1te 'prepare_samples' kind-prepare_samples/instance-qyxpumun 'Job' kind-Job/instance-rb7logwp 'merge_phlat_calls' kind-merge_phlat_calls/instance-6j0cplsq 'get_all_tool_inputs' kind-parse_config_file/instance-utcdr1zk 'run_bwa' kind-EncapsulatedJob/instance-gxt8is74 'run_cutadapt' kind-EncapsulatedJob/instance-o9p20i2e 'run_mutect' kind-EncapsulatedJob/instance-bj0twc34 'Job' kind-Job/instance-770lc4lu 'run_bwa' kind-EncapsulatedJob/instance-5drwan9s 'run_mutation_aggregator' kind-EncapsulatedJob/instance-_w1f8gtw 'Job' kind-Job/instance-ccadqh9a 'run_phlat' kind-EncapsulatedJob/instance-tm0pcusm 'run_transgene' kind-run_transgene/instance-qj3j385j 'Job' kind-Job/instance-l6jndejt 'run_phlat' kind-EncapsulatedJob/instance-p8idulz4 'Job' kind-Job/instance-7rcs33og 'run_star' kind-EncapsulatedJob/instance-y_lo_bvx 'Job' kind-Job/instance-p3i6d1f0 'Job' kind-Job/instance-19ln8xui 'get_patient_fastqs' kind-get_patient_fastqs/instance-2pjao3y3 'launch_protect' kind-launch_protect/instance-oyefziao 'get_patient_fastqs' kind-get_patient_fastqs/instance-ttmsgsd1 'Job' kind-Job/instance-j3z8ggs3 'run_phlat' kind-EncapsulatedJob/instance-czlzbjjr 'run_snpeff' kind-EncapsulatedJob/instance-1rgbxdn2 'get_patient_fastqs' kind-get_patient_fastqs/instance-0csdd7bl 'run_rsem' kind-EncapsulatedJob/instance-i6wy4k24 'Job' kind-Job/instance-na5nlnmq 'run_fusion' kind-EncapsulatedJob/instance-mgdq08hc 'Job' kind-Job/instance-bn2q7ksu 'Job' kind-Job/instance-5x2sz54b 'run_strelka_full' kind-EncapsulatedJob/instance-ixd9le0x 'run_indel_caller' kind-run_indel_caller/instance-0_hyd44y 'Job' kind-Job/instance-rong8s8v 'run_somaticsniper' kind-EncapsulatedJob/instance-wlu1vdpy 'Job' kind-Job/instance-hlsgfes5 'run_radia' kind-EncapsulatedJob/instance-y0q8img4 'Job' kind-Job/instance-p8u5n56k +Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/bin/ProTECT", line 11, in + load_entry_point('protect', 'console_scripts', 'ProTECT')() + File "/private/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1028, in main + Job.Runner.startToil(start, params) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 798, in startToil + return toil.restart() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 878, in restart + return self._runMainLoop(rootJobGraph) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/common.py", line 1138, in _runMainLoop + jobCache=self._jobCache).run() + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/leader.py", line 269, in run + raise FailedJobsException(self.config.jobStore, self.toilState.totalFailedJobs, self.jobStore) +toil.leader.FailedJobsException: The job store 'file:/scratch/drkthomp/jobStore' contains 42 failed jobs: 'run_muse' kind-EncapsulatedJob/instance-qi2fqyz_, 'run_transgene' kind-EncapsulatedJob/instance-vevukocf, 'Job' kind-Job/instance-7s1jyb6z, 'Job' kind-Job/instance-wizon1te, 'prepare_samples' kind-prepare_samples/instance-qyxpumun, 'Job' kind-Job/instance-rb7logwp, 'merge_phlat_calls' kind-merge_phlat_calls/instance-6j0cplsq, 'get_all_tool_inputs' kind-parse_config_file/instance-utcdr1zk, 'run_bwa' kind-EncapsulatedJob/instance-gxt8is74, 'run_cutadapt' kind-EncapsulatedJob/instance-o9p20i2e, 'run_mutect' kind-EncapsulatedJob/instance-bj0twc34, 'Job' kind-Job/instance-770lc4lu, 'run_bwa' kind-EncapsulatedJob/instance-5drwan9s, 'run_mutation_aggregator' kind-EncapsulatedJob/instance-_w1f8gtw, 'Job' kind-Job/instance-ccadqh9a, 'run_phlat' kind-EncapsulatedJob/instance-tm0pcusm, 'run_transgene' kind-run_transgene/instance-qj3j385j, 'Job' kind-Job/instance-l6jndejt, 'run_phlat' kind-EncapsulatedJob/instance-p8idulz4, 'Job' kind-Job/instance-7rcs33og, 'run_star' kind-EncapsulatedJob/instance-y_lo_bvx, 'Job' kind-Job/instance-p3i6d1f0, 'Job' kind-Job/instance-19ln8xui, 'get_patient_fastqs' kind-get_patient_fastqs/instance-2pjao3y3, 'launch_protect' kind-launch_protect/instance-oyefziao, 'get_patient_fastqs' kind-get_patient_fastqs/instance-ttmsgsd1, 'Job' kind-Job/instance-j3z8ggs3, 'run_phlat' kind-EncapsulatedJob/instance-czlzbjjr, 'run_snpeff' kind-EncapsulatedJob/instance-1rgbxdn2, 'get_patient_fastqs' kind-get_patient_fastqs/instance-0csdd7bl, 'run_rsem' kind-EncapsulatedJob/instance-i6wy4k24, 'Job' kind-Job/instance-na5nlnmq, 'run_fusion' kind-EncapsulatedJob/instance-mgdq08hc, 'Job' kind-Job/instance-bn2q7ksu, 'Job' kind-Job/instance-5x2sz54b, 'run_strelka_full' kind-EncapsulatedJob/instance-ixd9le0x, 'run_indel_caller' kind-run_indel_caller/instance-0_hyd44y, 'Job' kind-Job/instance-rong8s8v, 'run_somaticsniper' kind-EncapsulatedJob/instance-wlu1vdpy, 'Job' kind-Job/instance-hlsgfes5, 'run_radia' kind-EncapsulatedJob/instance-y0q8img4, 'Job' kind-Job/instance-p8u5n56k +Log from job 'run_transgene' kind-run_transgene/instance-qj3j385j follows: +=========> + [2020-11-27T19:37:40-0800] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-11-27T19:37:40-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. + usage: transgene [-h] [--peptides PEPTIDE_FILE] + [--snpeff SNPEFF_FILE | --vep VEP_FILE] + [--genome GENOME_FILE] [--indel_extend_length EXTEND_LENGTH] + [--fusions FUSION_FILE] [--transcripts TRANSCRIPT_FILE] + [--annotation ANNOTATION_FILE] [--filter_mt_fusions] + [--filter_ig_pairs] [--filter_rna_gene_fusions] + [--filter_readthroughs] + [--readthrough_threshold RT_THRESHOLD] [--rna_file RNA_FILE] + [--reject_threshold REJECT_THRESHOLD] + [--min_rna_alt_freq RNA_MIN_ALT_FREQ] [--filterOxoG] + [--dna_file DNA_FILE] + [--min_OxoG_variant_freq OXOG_MIN_ALT_FREQ] + [--log_level {INFO,DEBUG,WARNING,ERROR}] [--log_file LOGFILE] + --prefix PREFIX [--pep_lens PEP_LENS] [--no_json_dumps] + [--cores CORES] + transgene: error: argument --cores: invalid int value: '80.0' + [2020-11-27T19:38:46-0800] [MainThread] [W] [toil.fileStores.abstractFileStore] LOG-TO-MASTER: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-vevukocf/cleanup/file-rasmyjg9/stream used 1817.94% (5.5 GB [5914071040B] used, 310.2 MB [325317817B] requested) at the end of its run. + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/src/protect/common.py", line 137, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib64/python3.6/subprocess.py", line 291, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpeek2yuwe/a6ab82f8-4fb2-47c7-aca6-8dc94f9073ff:/data', '--log-driver=none', 'aarjunrao/transgene:2.5.0', '--peptides', '/data/gencode.v25.pc_translations_NOPARY.fa', '--prefix', 'transgened', '--pep_lens', '9,10,15', '--cores', '80.0', '--genome', '/data/hg38.fa', '--annotation', '/data/gencode.v25.annotation_NOPARY.gtf', '--snpeff', '/data/snpeffed_muts.vcf', '--rna_file', '/data/rna.bam', '--transcripts', '/data/gencode.v25.pc_transcripts_NOPARY.fa', '--fusions', '/data/fusion_calls']' returned non-zero exit status 2. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/private/home/drkthomp/protect3/venv/lib/python3.6/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/private/home/drkthomp/protect3/src/protect/mutation_translation.py", line 118, in run_transgene + tool_version=transgene_options['version']) + File "/private/home/drkthomp/protect3/src/protect/common.py", line 140, in docker_call + 'for command \"%s\"' % ' '.join(call),) + RuntimeError: docker command returned a non-zero exit status (2)for command "docker run --rm=true -v /data/scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpeek2yuwe/a6ab82f8-4fb2-47c7-aca6-8dc94f9073ff:/data --log-driver=none aarjunrao/transgene:2.5.0 --peptides /data/gencode.v25.pc_translations_NOPARY.fa --prefix transgened --pep_lens 9,10,15 --cores 80.0 --genome /data/hg38.fa --annotation /data/gencode.v25.annotation_NOPARY.gtf --snpeff /data/snpeffed_muts.vcf --rna_file /data/rna.bam --transcripts /data/gencode.v25.pc_transcripts_NOPARY.fa --fusions /data/fusion_calls" + [2020-11-27T19:38:46-0800] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host mustard +<========= diff --git a/errors/2020-11-27-20-37-31.txt b/errors/2020-11-27-20-37-31.txt new file mode 100644 index 00000000..6da57ce6 --- /dev/null +++ b/errors/2020-11-27-20-37-31.txt @@ -0,0 +1,246 @@ +[2020-11-27T20:37:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_transgene/instance-qj3j385j +[2020-11-27T20:37:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-_w1f8gtw +[2020-11-27T20:37:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-1rgbxdn2 +[2020-11-27T20:37:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-EncapsulatedJob/instance-bj0twc34 +[2020-11-27T20:37:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-Job/instance-l6jndejt +[2020-11-27T20:37:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-Job/instance-hlsgfes5 +[2020-11-27T20:37:33-0800] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-Job/instance-bn2q7ksu +[2020-11-27T20:37:33-0800] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host mustard. +[2020-11-27T20:37:33-0800] [MainThread] [I] [toil.leader] Issued job 'run_transgene' kind-run_transgene/instance-qj3j385j with job batch system ID: 0 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T20:37:33-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpmmjr0k4x/worker_log.txt +[2020-11-27T20:37:35-0800] [MainThread] [I] [toil.leader] 1 jobs are running, 0 jobs are issued and waiting to run +[2020-11-27T20:45:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:45:50: Exporting files/for-job/kind-run_transgene/instance-qj3j385j/file-86chipdx/transgened_tumor_9_mer_peptides.faa to output location +[2020-11-27T20:45:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:45:50: Exporting files/for-job/kind-run_transgene/instance-qj3j385j/file-xruawzb2/transgened_normal_9_mer_peptides.faa to output location +[2020-11-27T20:45:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:45:50: Exporting files/for-job/kind-run_transgene/instance-qj3j385j/file-26c0ic8p/transgened_tumor_9_mer_peptides.faa.map to output location +[2020-11-27T20:45:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:45:50: Exporting files/for-job/kind-run_transgene/instance-qj3j385j/file-7td9w2uo/transgened_tumor_10_mer_peptides.faa to output location +[2020-11-27T20:45:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:45:50: Exporting files/for-job/kind-run_transgene/instance-qj3j385j/file-7w39luru/transgened_normal_10_mer_peptides.faa to output location +[2020-11-27T20:45:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:45:50: Exporting files/for-job/kind-run_transgene/instance-qj3j385j/file-8ujpjbxn/transgened_tumor_10_mer_peptides.faa.map to output location +[2020-11-27T20:45:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:45:50: Exporting files/for-job/kind-run_transgene/instance-qj3j385j/file-14zqqnaa/transgened_tumor_15_mer_peptides.faa to output location +[2020-11-27T20:45:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:45:50: Exporting files/for-job/kind-run_transgene/instance-qj3j385j/file-4jtfw7i_/transgened_normal_15_mer_peptides.faa to output location +[2020-11-27T20:45:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:45:50: Exporting files/for-job/kind-run_transgene/instance-qj3j385j/file-wk6ympyl/transgened_tumor_15_mer_peptides.faa.map to output location +[2020-11-27T20:45:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:45:50: Exporting files/for-job/kind-run_transgene/instance-qj3j385j/file-x93455ry/mutations.vcf to output location +[2020-11-27T20:45:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:45:50: Exporting files/for-job/kind-run_transgene/instance-qj3j385j/file-a2h4vq6b/fusions.bedpe to output location +[2020-11-27T20:45:50-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:45:50: Ran transgene on TEST successfully +[2020-11-27T20:45:50-0800] [Thread-3 ] [W] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:45:50: Job used more disk than requested. Consider modifying the user script to avoid the chance of failure due to incorrectly requested resources. Job files/for-job/kind-EncapsulatedJob/instance-vevukocf/cleanup/file-rasmyjg9/stream used 1817.95% (5.5 GB [5914116096B] used, 310.2 MB [325317817B] requested) at the end of its run. +[2020-11-27T20:45:51-0800] [MainThread] [I] [toil.leader] Job ended: 'run_transgene' kind-run_transgene/instance-qj3j385j +[2020-11-27T20:45:51-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-qo9wav0t with job batch system ID: 1 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:45:52-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1wytcmci/worker_log.txt +[2020-11-27T20:45:52-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-qo9wav0t +[2020-11-27T20:45:52-0800] [MainThread] [I] [toil.leader] Issued job 'delete_bams' kind-delete_bams/instance-7d_al09s with job batch system ID: 2 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T20:45:52-0800] [MainThread] [I] [toil.leader] Issued job 'EncapsulatedJob' kind-EncapsulatedJob/instance-hw92m7bh with job batch system ID: 3 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T20:45:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpictoor99/worker_log.txt +[2020-11-27T20:45:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpbyrnyojh/worker_log.txt +[2020-11-27T20:45:53-0800] [MainThread] [I] [toil.leader] Job ended: 'delete_bams' kind-delete_bams/instance-7d_al09s +[2020-11-27T20:45:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:45:54: Deleting "rna_transcriptome.bam" for patient "TEST". +[2020-11-27T20:45:54-0800] [MainThread] [I] [toil.leader] Job ended: 'EncapsulatedJob' kind-EncapsulatedJob/instance-hw92m7bh +[2020-11-27T20:45:54-0800] [MainThread] [I] [toil.leader] Issued job 'predict_mhci_binding' kind-predict_mhci_binding/instance-455ncw2b with job batch system ID: 4 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T20:45:54-0800] [MainThread] [I] [toil.leader] Issued job 'predict_mhci_binding' kind-predict_mhci_binding/instance-9n5evjis with job batch system ID: 5 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T20:45:54-0800] [MainThread] [I] [toil.leader] Issued job 'predict_mhci_binding' kind-predict_mhci_binding/instance-9kezlcrr with job batch system ID: 6 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T20:45:54-0800] [MainThread] [I] [toil.leader] Issued job 'predict_mhci_binding' kind-predict_mhci_binding/instance-1hg8snpn with job batch system ID: 7 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T20:45:54-0800] [MainThread] [I] [toil.leader] Issued job 'predict_mhci_binding' kind-predict_mhci_binding/instance-c701ctec with job batch system ID: 8 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T20:45:54-0800] [MainThread] [I] [toil.leader] Issued job 'predict_mhci_binding' kind-predict_mhci_binding/instance-kqs14gzp with job batch system ID: 9 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T20:45:54-0800] [MainThread] [I] [toil.leader] Issued job 'predict_mhci_binding' kind-predict_mhci_binding/instance-r6cwbg4s with job batch system ID: 10 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T20:45:54-0800] [MainThread] [I] [toil.leader] Issued job 'predict_mhci_binding' kind-predict_mhci_binding/instance-pkjdd66b with job batch system ID: 11 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T20:45:54-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:45:54: Ran spawn_anti on TEST successfully +[2020-11-27T20:45:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpasrz42ma/worker_log.txt +[2020-11-27T20:45:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpwa3xqzpc/worker_log.txt +[2020-11-27T20:45:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpd27gxj7p/worker_log.txt +[2020-11-27T20:45:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp6pkw9lyv/worker_log.txt +[2020-11-27T20:45:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp2if5xl90/worker_log.txt +[2020-11-27T20:45:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1q6dkfzr/worker_log.txt +[2020-11-27T20:45:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp48bh__b3/worker_log.txt +[2020-11-27T20:45:55-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpsm0t6tn2/worker_log.txt +[2020-11-27T20:46:17-0800] [MainThread] [I] [toil.leader] Job ended: 'predict_mhci_binding' kind-predict_mhci_binding/instance-c701ctec +[2020-11-27T20:46:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:17: Ran mhci on TEST:HLA-C*06:02:9 successfully +[2020-11-27T20:46:17-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:17: Ran predict_normal_binding on TEST for allele HLA-C*06:02 and length 9 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran mhci on TEST:HLA-C*06:02:10 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran predict_normal_binding on TEST for allele HLA-C*06:02 and length 10 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran mhci on TEST:HLA-C*06:02:10 successfully +[2020-11-27T20:46:20-0800] [MainThread] [I] [toil.leader] Job ended: 'predict_mhci_binding' kind-predict_mhci_binding/instance-kqs14gzp +[2020-11-27T20:46:20-0800] [MainThread] [I] [toil.leader] Job ended: 'predict_mhci_binding' kind-predict_mhci_binding/instance-9n5evjis +[2020-11-27T20:46:20-0800] [MainThread] [I] [toil.leader] Job ended: 'predict_mhci_binding' kind-predict_mhci_binding/instance-r6cwbg4s +[2020-11-27T20:46:20-0800] [MainThread] [I] [toil.leader] Job ended: 'predict_mhci_binding' kind-predict_mhci_binding/instance-1hg8snpn +[2020-11-27T20:46:20-0800] [MainThread] [I] [toil.leader] Job ended: 'predict_mhci_binding' kind-predict_mhci_binding/instance-pkjdd66b +[2020-11-27T20:46:20-0800] [MainThread] [I] [toil.leader] Job ended: 'predict_mhci_binding' kind-predict_mhci_binding/instance-9kezlcrr +[2020-11-27T20:46:20-0800] [MainThread] [I] [toil.leader] Job ended: 'predict_mhci_binding' kind-predict_mhci_binding/instance-455ncw2b +[2020-11-27T20:46:20-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ueveifko with job batch system ID: 12 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran mhci on TEST:HLA-C*07:01:10 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran predict_normal_binding on TEST for allele HLA-C*07:01 and length 10 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran mhci on TEST:HLA-C*07:01:10 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran mhci on TEST:HLA-A*29:02:10 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran predict_normal_binding on TEST for allele HLA-A*29:02 and length 10 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran mhci on TEST:HLA-A*29:02:10 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran mhci on TEST:HLA-C*07:01:9 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran predict_normal_binding on TEST for allele HLA-C*07:01 and length 9 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran mhci on TEST:HLA-C*07:01:9 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran mhci on TEST:HLA-B*45:01:10 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran predict_normal_binding on TEST for allele HLA-B*45:01 and length 10 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran mhci on TEST:HLA-B*45:01:10 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran mhci on TEST:HLA-B*45:01:9 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran predict_normal_binding on TEST for allele HLA-B*45:01 and length 9 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran mhci on TEST:HLA-B*45:01:9 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran mhci on TEST:HLA-A*29:02:9 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran predict_normal_binding on TEST for allele HLA-A*29:02 and length 9 successfully +[2020-11-27T20:46:20-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:20: Ran mhci on TEST:HLA-A*29:02:9 successfully +[2020-11-27T20:46:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpjn710ty0/worker_log.txt +[2020-11-27T20:46:21-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ueveifko +[2020-11-27T20:46:21-0800] [MainThread] [I] [toil.leader] Issued job 'merge_mhc_peptide_calls' kind-merge_mhc_peptide_calls/instance-08brkiy3 with job batch system ID: 13 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T20:46:21-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmph3l0vvz2/worker_log.txt +[2020-11-27T20:46:22-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:22: Merging MHC calls +[2020-11-27T20:46:22-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:22: Exporting files/for-job/kind-merge_mhc_peptide_calls/instance-08brkiy3/file-c8k76jme/mhci_merged_files.list to output location +[2020-11-27T20:46:22-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:22: Exporting files/for-job/kind-merge_mhc_peptide_calls/instance-08brkiy3/file-i6uyn1ks/mhcii_merged_files.list to output location +[2020-11-27T20:46:22-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_mhc_peptide_calls' kind-merge_mhc_peptide_calls/instance-08brkiy3 +[2020-11-27T20:46:22-0800] [MainThread] [I] [toil.leader] Issued job 'wrap_rankboost' kind-wrap_rankboost/instance-wu7k02h6 with job batch system ID: 14 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T20:46:23-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpk64kqmq7/worker_log.txt +[2020-11-27T20:46:23-0800] [MainThread] [I] [toil.leader] Job ended: 'wrap_rankboost' kind-wrap_rankboost/instance-wu7k02h6 +[2020-11-27T20:46:23-0800] [MainThread] [I] [toil.leader] Issued job 'boost_ranks' kind-boost_ranks/instance-vg6lyis2 with job batch system ID: 15 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T20:46:23-0800] [MainThread] [I] [toil.leader] Issued job 'email_report' kind-email_report/instance-5_kgos1n with job batch system ID: 16 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T20:46:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpz_sp75cs/worker_log.txt +[2020-11-27T20:46:24-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp6ac4pw6p/worker_log.txt +[2020-11-27T20:46:24-0800] [MainThread] [I] [toil.leader] Job ended: 'email_report' kind-email_report/instance-5_kgos1n +[2020-11-27T20:46:38-0800] [MainThread] [I] [toil.leader] Job ended: 'boost_ranks' kind-boost_ranks/instance-vg6lyis2 +[2020-11-27T20:46:38-0800] [MainThread] [I] [toil.leader] Issued job 'wrap_rankboost' kind-wrap_rankboost/instance-wu7k02h6 with job batch system ID: 17 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T20:46:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:39: Exporting files/for-job/kind-boost_ranks/instance-vg6lyis2/file-xof95vzj/mhci_rankboost_concise_results.tsv to output location +[2020-11-27T20:46:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:39: Exporting files/for-job/kind-boost_ranks/instance-vg6lyis2/file-uchb2up2/mhci_rankboost_detailed_results.txt to output location +[2020-11-27T20:46:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:39: Exporting files/for-job/kind-boost_ranks/instance-vg6lyis2/file-ic5jgau_/mhcii_rankboost_concise_results.tsv to output location +[2020-11-27T20:46:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:39: Exporting files/for-job/kind-boost_ranks/instance-vg6lyis2/file-pgtov982/mhcii_rankboost_detailed_results.txt to output location +[2020-11-27T20:46:39-0800] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 11-27-2020 20:46:39: Ran boost_ranks on TEST successfully +[2020-11-27T20:46:39-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp9sinnbx0/worker_log.txt +[2020-11-27T20:46:39-0800] [MainThread] [I] [toil.leader] Job ended: 'wrap_rankboost' kind-wrap_rankboost/instance-wu7k02h6 +[2020-11-27T20:46:39-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-j3z8ggs3 with job batch system ID: 18 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:46:39-0800] [MainThread] [I] [toil.leader] Issued job 'merge_mhc_peptide_calls' kind-merge_mhc_peptide_calls/instance-08brkiy3 with job batch system ID: 19 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T20:46:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmptou23lm_/worker_log.txt +[2020-11-27T20:46:40-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp_1b6g6t8/worker_log.txt +[2020-11-27T20:46:40-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_mhc_peptide_calls' kind-merge_mhc_peptide_calls/instance-08brkiy3 +[2020-11-27T20:46:40-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ueveifko with job batch system ID: 20 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:46:40-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-j3z8ggs3 +[2020-11-27T20:46:40-0800] [MainThread] [I] [toil.leader] Issued job 'run_rsem' kind-EncapsulatedJob/instance-i6wy4k24 with job batch system ID: 21 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-27T20:46:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpm9oglyjg/worker_log.txt +[2020-11-27T20:46:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp1d0vra6w/worker_log.txt +[2020-11-27T20:46:41-0800] [MainThread] [I] [toil.leader] Job ended: 'run_rsem' kind-EncapsulatedJob/instance-i6wy4k24 +[2020-11-27T20:46:41-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ueveifko +[2020-11-27T20:46:41-0800] [MainThread] [I] [toil.leader] Issued job 'spawn_antigen_predictors' kind-EncapsulatedJob/instance-hw92m7bh with job batch system ID: 22 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T20:46:41-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpd0feznzd/worker_log.txt +[2020-11-27T20:46:41-0800] [MainThread] [I] [toil.leader] Job ended: 'spawn_antigen_predictors' kind-EncapsulatedJob/instance-hw92m7bh +[2020-11-27T20:46:41-0800] [MainThread] [I] [toil.leader] Issued job 'merge_phlat_calls' kind-merge_phlat_calls/instance-6j0cplsq with job batch system ID: 23 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T20:46:41-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-qo9wav0t with job batch system ID: 24 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:46:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp88vspm0v/worker_log.txt +[2020-11-27T20:46:42-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpb_4sgugw/worker_log.txt +[2020-11-27T20:46:42-0800] [MainThread] [I] [toil.leader] Job ended: 'merge_phlat_calls' kind-merge_phlat_calls/instance-6j0cplsq +[2020-11-27T20:46:42-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-7s1jyb6z with job batch system ID: 25 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:46:42-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-5x2sz54b with job batch system ID: 26 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:46:42-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-p8u5n56k with job batch system ID: 27 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:46:42-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-qo9wav0t +[2020-11-27T20:46:42-0800] [MainThread] [I] [toil.leader] Issued job 'run_transgene' kind-EncapsulatedJob/instance-vevukocf with job batch system ID: 28 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T20:46:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpogs_b6sg/worker_log.txt +[2020-11-27T20:46:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpd54o9deb/worker_log.txt +[2020-11-27T20:46:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpyxfd10jz/worker_log.txt +[2020-11-27T20:46:43-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp3gq7g_sm/worker_log.txt +[2020-11-27T20:46:43-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-5x2sz54b +[2020-11-27T20:46:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-EncapsulatedJob/instance-p8idulz4 with job batch system ID: 29 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T20:46:43-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-7s1jyb6z +[2020-11-27T20:46:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-EncapsulatedJob/instance-czlzbjjr with job batch system ID: 30 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T20:46:43-0800] [MainThread] [I] [toil.leader] Job ended: 'run_transgene' kind-EncapsulatedJob/instance-vevukocf +[2020-11-27T20:46:43-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-p3i6d1f0 with job batch system ID: 31 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:46:43-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-770lc4lu with job batch system ID: 32 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:46:43-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-p8u5n56k +[2020-11-27T20:46:43-0800] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-EncapsulatedJob/instance-tm0pcusm with job batch system ID: 33 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T20:46:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpynnkik02/worker_log.txt +[2020-11-27T20:46:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpf5tasji5/worker_log.txt +[2020-11-27T20:46:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpihfg78cx/worker_log.txt +[2020-11-27T20:46:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp958qbs9w/worker_log.txt +[2020-11-27T20:46:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_phlat' kind-EncapsulatedJob/instance-p8idulz4 +[2020-11-27T20:46:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_phlat' kind-EncapsulatedJob/instance-czlzbjjr +[2020-11-27T20:46:44-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-p3i6d1f0 +[2020-11-27T20:46:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_fusion' kind-EncapsulatedJob/instance-mgdq08hc with job batch system ID: 34 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T20:46:44-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-770lc4lu +[2020-11-27T20:46:44-0800] [MainThread] [I] [toil.leader] Issued job 'run_snpeff' kind-EncapsulatedJob/instance-1rgbxdn2 with job batch system ID: 35 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T20:46:44-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp590xfcam/worker_log.txt +[2020-11-27T20:46:44-0800] [MainThread] [I] [toil.leader] Job ended: 'run_phlat' kind-EncapsulatedJob/instance-tm0pcusm +[2020-11-27T20:46:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp6yr0p07w/worker_log.txt +[2020-11-27T20:46:45-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmptz19qpf7/worker_log.txt +[2020-11-27T20:46:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_fusion' kind-EncapsulatedJob/instance-mgdq08hc +[2020-11-27T20:46:45-0800] [MainThread] [I] [toil.leader] Job ended: 'run_snpeff' kind-EncapsulatedJob/instance-1rgbxdn2 +[2020-11-27T20:46:45-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-bn2q7ksu with job batch system ID: 36 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:46:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpo5bjfie1/worker_log.txt +[2020-11-27T20:46:46-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-bn2q7ksu +[2020-11-27T20:46:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutation_aggregator' kind-EncapsulatedJob/instance-_w1f8gtw with job batch system ID: 37 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T20:46:46-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmps2cl_yof/worker_log.txt +[2020-11-27T20:46:46-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutation_aggregator' kind-EncapsulatedJob/instance-_w1f8gtw +[2020-11-27T20:46:46-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-rong8s8v with job batch system ID: 38 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:46:46-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-ccadqh9a with job batch system ID: 39 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:46:46-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-na5nlnmq with job batch system ID: 40 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:46:46-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-rb7logwp with job batch system ID: 41 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:46:46-0800] [MainThread] [I] [toil.leader] Issued job 'run_indel_caller' kind-run_indel_caller/instance-0_hyd44y with job batch system ID: 42 and cores: 1, disk: 100.0 M, and memory: 100.0 M +[2020-11-27T20:46:46-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-7rcs33og with job batch system ID: 43 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:46:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmphz7t7at1/worker_log.txt +[2020-11-27T20:46:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpinm3ewce/worker_log.txt +[2020-11-27T20:46:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpi60zi220/worker_log.txt +[2020-11-27T20:46:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpi6_xklus/worker_log.txt +[2020-11-27T20:46:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpfmcma03t/worker_log.txt +[2020-11-27T20:46:47-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-rong8s8v +[2020-11-27T20:46:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_mutect' kind-EncapsulatedJob/instance-bj0twc34 with job batch system ID: 44 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-27T20:46:47-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-7rcs33og +[2020-11-27T20:46:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_radia' kind-EncapsulatedJob/instance-y0q8img4 with job batch system ID: 45 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-27T20:46:47-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpmxsika9t/worker_log.txt +[2020-11-27T20:46:47-0800] [MainThread] [I] [toil.leader] Job ended: 'run_indel_caller' kind-run_indel_caller/instance-0_hyd44y +[2020-11-27T20:46:47-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-ccadqh9a +[2020-11-27T20:46:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_muse' kind-EncapsulatedJob/instance-qi2fqyz_ with job batch system ID: 46 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T20:46:47-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-na5nlnmq +[2020-11-27T20:46:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_somaticsniper' kind-EncapsulatedJob/instance-wlu1vdpy with job batch system ID: 47 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T20:46:47-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-rb7logwp +[2020-11-27T20:46:47-0800] [MainThread] [I] [toil.leader] Issued job 'run_strelka_full' kind-EncapsulatedJob/instance-ixd9le0x with job batch system ID: 48 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T20:46:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpxf6ly6y0/worker_log.txt +[2020-11-27T20:46:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpvivhdx76/worker_log.txt +[2020-11-27T20:46:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_mutect' kind-EncapsulatedJob/instance-bj0twc34 +[2020-11-27T20:46:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpad9q95b4/worker_log.txt +[2020-11-27T20:46:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_radia' kind-EncapsulatedJob/instance-y0q8img4 +[2020-11-27T20:46:48-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-19ln8xui with job batch system ID: 49 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:46:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpncp1wg93/worker_log.txt +[2020-11-27T20:46:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_muse' kind-EncapsulatedJob/instance-qi2fqyz_ +[2020-11-27T20:46:48-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp5tj003wj/worker_log.txt +[2020-11-27T20:46:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_somaticsniper' kind-EncapsulatedJob/instance-wlu1vdpy +[2020-11-27T20:46:48-0800] [MainThread] [I] [toil.leader] Job ended: 'run_strelka_full' kind-EncapsulatedJob/instance-ixd9le0x +[2020-11-27T20:46:48-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-l6jndejt with job batch system ID: 50 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:46:48-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-hlsgfes5 with job batch system ID: 51 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:46:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpry2w5abd/worker_log.txt +[2020-11-27T20:46:49-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-19ln8xui +[2020-11-27T20:46:49-0800] [MainThread] [I] [toil.leader] Issued job 'run_star' kind-EncapsulatedJob/instance-y_lo_bvx with job batch system ID: 52 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-27T20:46:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpl4geon__/worker_log.txt +[2020-11-27T20:46:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpeelbgh50/worker_log.txt +[2020-11-27T20:46:49-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-l6jndejt +[2020-11-27T20:46:49-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-EncapsulatedJob/instance-5drwan9s with job batch system ID: 53 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-27T20:46:49-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-hlsgfes5 +[2020-11-27T20:46:49-0800] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-EncapsulatedJob/instance-gxt8is74 with job batch system ID: 54 and cores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-11-27T20:46:49-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpi1aq8x3g/worker_log.txt +[2020-11-27T20:46:50-0800] [MainThread] [I] [toil.leader] Job ended: 'run_star' kind-EncapsulatedJob/instance-y_lo_bvx +[2020-11-27T20:46:50-0800] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-wizon1te with job batch system ID: 55 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-11-27T20:46:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp7dg61whh/worker_log.txt +[2020-11-27T20:46:50-0800] [MainThread] [I] [toil.leader] Job ended: 'run_bwa' kind-EncapsulatedJob/instance-5drwan9s +[2020-11-27T20:46:50-0800] [MainThread] [I] [toil.leader] Issued job 'get_patient_fastqs' kind-get_patient_fastqs/instance-0csdd7bl with job batch system ID: 56 and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-11-27T20:46:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpjcszgql1/worker_log.txt +[2020-11-27T20:46:50-0800] [MainThread] [I] [toil.leader] Job ended: 'run_bwa' kind-EncapsulatedJob/instance-gxt8is74 +[2020-11-27T20:46:50-0800] [MainThread] [I] [toil.leader] Issued job 'get_patient_fastqs' kind-get_patient_fastqs/instance-2pjao3y3 with job batch system ID: 57 and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-11-27T20:46:50-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpxx7a34n9/worker_log.txt +[2020-11-27T20:46:50-0800] [MainThread] [I] [toil.leader] Job ended: 'Job' kind-Job/instance-wizon1te +[2020-11-27T20:46:50-0800] [MainThread] [I] [toil.leader] Issued job 'run_cutadapt' kind-EncapsulatedJob/instance-o9p20i2e with job batch system ID: 58 and cores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-11-27T20:46:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpsfl2zr_d/worker_log.txt +[2020-11-27T20:46:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpr0_t11ju/worker_log.txt +[2020-11-27T20:46:51-0800] [MainThread] [I] [toil.leader] Job ended: 'get_patient_fastqs' kind-get_patient_fastqs/instance-0csdd7bl +[2020-11-27T20:46:51-0800] [MainThread] [I] [toil.leader] Job ended: 'get_patient_fastqs' kind-get_patient_fastqs/instance-2pjao3y3 +[2020-11-27T20:46:51-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpyn2l6gkn/worker_log.txt +[2020-11-27T20:46:51-0800] [MainThread] [I] [toil.leader] Job ended: 'run_cutadapt' kind-EncapsulatedJob/instance-o9p20i2e +[2020-11-27T20:46:51-0800] [MainThread] [I] [toil.leader] Issued job 'get_patient_fastqs' kind-get_patient_fastqs/instance-ttmsgsd1 with job batch system ID: 59 and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-11-27T20:46:52-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmp9w9zamvd/worker_log.txt +[2020-11-27T20:46:52-0800] [MainThread] [I] [toil.leader] Job ended: 'get_patient_fastqs' kind-get_patient_fastqs/instance-ttmsgsd1 +[2020-11-27T20:46:52-0800] [MainThread] [I] [toil.leader] Issued job 'prepare_samples' kind-prepare_samples/instance-qyxpumun with job batch system ID: 60 and cores: 1, disk: 40.0 G, and memory: 2.0 G +[2020-11-27T20:46:53-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpdxfhjxb8/worker_log.txt +[2020-11-27T20:46:53-0800] [MainThread] [I] [toil.leader] Job ended: 'prepare_samples' kind-prepare_samples/instance-qyxpumun +[2020-11-27T20:46:53-0800] [MainThread] [I] [toil.leader] Issued job 'launch_protect' kind-launch_protect/instance-oyefziao with job batch system ID: 61 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T20:46:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpj442pakx/worker_log.txt +[2020-11-27T20:46:54-0800] [MainThread] [I] [toil.leader] Job ended: 'launch_protect' kind-launch_protect/instance-oyefziao +[2020-11-27T20:46:54-0800] [MainThread] [I] [toil.leader] Issued job 'get_all_tool_inputs' kind-parse_config_file/instance-utcdr1zk with job batch system ID: 62 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-11-27T20:46:54-0800] [MainThread] [I] [toil.worker] Redirecting logging to /scratch/drkthomp/workDir/node-bd51875c-dc23-4882-b1fa-2fa41c9dc75d-e8611b53-bdab-4598-acae-87977f99831e/tmpv3ofdhjs/worker_log.txt +[2020-11-27T20:46:54-0800] [MainThread] [I] [toil.leader] Job ended: 'get_all_tool_inputs' kind-parse_config_file/instance-utcdr1zk +[2020-11-27T20:46:57-0800] [MainThread] [I] [toil.leader] Finished toil run successfully. +[2020-11-27T20:47:10-0800] [MainThread] [I] [toil.common] Successfully deleted the job store: FileJobStore(/scratch/drkthomp/jobStore) diff --git a/errors/202010101748 Insufficient Resources b/errors/202010101748 Insufficient Resources new file mode 100644 index 00000000..9aeca8b7 --- /dev/null +++ b/errors/202010101748 Insufficient Resources @@ -0,0 +1,379 @@ +(venv) [drkthomp@drkthomp-virtualbox protect3]$ ProTECT --config ProTECT_config.yaml jobStore +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'parse_co +nfig_file' kind-parse_config_file/instance-k2c59tny with job batch system ID: +0 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpgrcts8ag/worker_log.txt +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Job ended: 'parse_co +nfig_file' kind-parse_config_file/instance-k2c59tny +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] 0 jobs are running, +0 jobs are issued and waiting to run +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-vf7ss4hd with job batch system +ID: 1 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-wf2i3yy9 with job batch system +ID: 2 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-l5qjsmi5 with job batch system +ID: 3 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-2kshu7ye with job batch system +ID: 4 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-nsthlmns with job batch system +ID: 5 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-mhlqarxw with job batch system +ID: 6 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-qajdh8q7 with job batch system +ID: 7 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-mst6z8ww with job batch system +ID: 8 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-g906kxrj with job batch system +ID: 9 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-r1py6hlp with job batch system +ID: 10 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-t3ypqu3g with job batch system +ID: 11 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-0ye9d5y5 with job batch system +ID: 12 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-1hk1njja with job batch system +ID: 13 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-_1ignq05 with job batch system +ID: 14 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-6ulo5c99 with job batch system +ID: 15 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-6d1172lv with job batch system +ID: 16 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-5zocomnu with job batch system +ID: 17 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-_k7jjrx8 with job batch system +ID: 18 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-otuezj6c with job batch system +ID: 19 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-p_c80i2p with job batch system +ID: 20 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-b4jsp5w6 with job batch system +ID: 21 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-8s5762r6 with job batch system +ID: 22 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-hbh2pi4k with job batch system +ID: 23 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-mh6rni3a with job batch system +ID: 24 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-f9rvn2n3 with job batch system +ID: 25 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-fyqt5509 with job batch system +ID: 26 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-2tq6z6e3 with job batch system +ID: 27 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-lljowav2 with job batch system +ID: 28 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [MainThread] [I] [toil.leader] Issued job 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-m73p1h9s with job batch system +ID: 29 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T16:29:03-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 16:29:03: Parsing config file +[2020-10-10T16:29:03-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 16:29:03: Obtaining tool inputs +[2020-10-10T16:29:03-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 16:29:03: Obtained tool inputs +[2020-10-10T16:29:04-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpiqosp6c8/worker_log.txt +[2020-10-10T16:29:04-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmptze_fdi_/worker_log.txt +[2020-10-10T16:29:04-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpjibucmzb/worker_log.txt +[2020-10-10T16:37:34-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 16:37:34: Obtaining file (rsem:index) to the file + job store +[2020-10-10T16:37:34-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-l5qjsmi5 +[2020-10-10T16:37:34-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpn_whjmy9/worker_log.txt +[2020-10-10T16:37:57-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-qajdh8q7 +[2020-10-10T16:37:57-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 16:37:57: Obtaining file (indexes:cosmic_vcf) to +the file job store +[2020-10-10T16:37:57-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpje957fcw/worker_log.txt +[2020-10-10T16:56:02-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-wf2i3yy9 +[2020-10-10T16:56:02-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 16:56:02: Obtaining file (bwa:index) to the file +job store +[2020-10-10T16:56:02-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmp2dopxo5i/worker_log.txt +[2020-10-10T16:56:03-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-mst6z8ww +[2020-10-10T16:56:03-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpi0txcbtj/worker_log.txt +[2020-10-10T16:56:03-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 16:56:03: Obtaining file (indexes:cosmic_idx) to +the file job store +[2020-10-10T16:56:03-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-2tq6z6e3 +[2020-10-10T16:56:04-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmp617xei8n/worker_log.txt +[2020-10-10T16:56:04-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 16:56:04: Obtaining file (reports:itx_resistance_ +file) to the file job store +[2020-10-10T16:56:04-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-f9rvn2n3 +[2020-10-10T16:56:04-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpajlz762o/worker_log.txt +[2020-10-10T16:56:04-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 16:56:04: Obtaining file (mhcii:method_file) to t +he file job store +[2020-10-10T17:03:26-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-2kshu7ye +[2020-10-10T17:03:26-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:03:26: Obtaining file (indexes:genome_fasta) t +o the file job store +[2020-10-10T17:03:26-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmp1np2pyv1/worker_log.txt +[2020-10-10T17:03:27-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-lljowav2 +[2020-10-10T17:03:27-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:03:27: Obtaining file (reports:immune_resistan +ce_pathways_file) to the file job store +[2020-10-10T17:03:27-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpuncfvor6/worker_log.txt +[2020-10-10T17:03:27-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-6ulo5c99 +[2020-10-10T17:03:27-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:03:27: Obtaining file (radia:pseudogene_beds) +to the file job store +[2020-10-10T17:03:27-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmp65msjn8v/worker_log.txt +[2020-10-10T17:03:28-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:03:28: Obtaining file (radia:gencode_beds) to +the file job store +[2020-10-10T17:03:28-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-6d1172lv +[2020-10-10T17:03:28-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpfgbxmw_9/worker_log.txt +[2020-10-10T17:03:29-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:03:29: Obtaining file (strelka:config_file) to + the file job store +[2020-10-10T17:03:29-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-5zocomnu +[2020-10-10T17:03:29-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmp_ycx77gt/worker_log.txt +[2020-10-10T17:03:29-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:03:29: Obtaining file (radia:retrogene_beds) t +o the file job store +[2020-10-10T17:03:29-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-_1ignq05 +[2020-10-10T17:03:29-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpq_ns7qry/worker_log.txt +[2020-10-10T17:03:30-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-fyqt5509 +[2020-10-10T17:03:30-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:03:30: Obtaining file (reports:mhc_pathways_fi +le) to the file job store +[2020-10-10T17:03:30-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmp70kw14rn/worker_log.txt +[2020-10-10T17:03:30-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-mh6rni3a +[2020-10-10T17:03:30-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:03:30: Obtaining file (mhci:method_file) to th +e file job store +[2020-10-10T17:03:30-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpl5etaoh7/worker_log.txt +[2020-10-10T17:07:11-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-hbh2pi4k +[2020-10-10T17:07:11-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpxlvvzbta/worker_log.txt +[2020-10-10T17:07:11-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:07:11: Obtaining file (phlat:index) to the fil +e job store +[2020-10-10T17:07:16-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-otuezj6c +[2020-10-10T17:07:17-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpa5zak7s2/worker_log.txt +[2020-10-10T17:07:17-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:07:17: Obtaining file (transgene:gencode_pepti +de_fasta) to the file job store +[2020-10-10T17:07:18-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-t3ypqu3g +[2020-10-10T17:07:18-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:07:18: Obtaining file (indexes:dbsnp_tbi) to t +he file job store +[2020-10-10T17:07:18-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmp_k_z8kmg/worker_log.txt +[2020-10-10T17:07:18-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-nsthlmns +[2020-10-10T17:07:18-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:07:18: Obtaining file (indexes:genome_fai) to +the file job store +[2020-10-10T17:07:18-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpr97iqosf/worker_log.txt +[2020-10-10T17:07:35-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-p_c80i2p +[2020-10-10T17:07:35-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:07:35: Obtaining file (transgene:gencode_trans +cript_fasta) to the file job store +[2020-10-10T17:07:35-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmp7_e0fyx_/worker_log.txt +[2020-10-10T17:07:48-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-0ye9d5y5 +[2020-10-10T17:07:48-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpa__uhdml/worker_log.txt +[2020-10-10T17:07:48-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:07:48: Obtaining file (radia:cosmic_beds) to t +he file job store +[2020-10-10T17:07:49-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-8s5762r6 +[2020-10-10T17:07:49-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpzylwfzh4/worker_log.txt +[2020-10-10T17:07:49-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:07:49: Obtaining file (transgene:genome_fasta) + to the file job store +[2020-10-10T17:16:16-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-_k7jjrx8 +[2020-10-10T17:16:16-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:16:16: Obtaining file (snpeff:index) to the fi +le job store +[2020-10-10T17:16:17-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpqj0c7_c5/worker_log.txt +[2020-10-10T17:16:17-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-m73p1h9s +[2020-10-10T17:16:17-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:16:17: Obtaining file (reports:car_t_targets_f +ile) to the file job store +[2020-10-10T17:16:17-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpw5dt_pus/worker_log.txt +[2020-10-10T17:19:37-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:19:37: Obtaining file (indexes:dbsnp_vcf) to t +he file job store +[2020-10-10T17:19:37-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-g906kxrj +[2020-10-10T17:19:38-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpb5x30dhe/worker_log.txt +[2020-10-10T17:19:39-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-r1py6hlp +[2020-10-10T17:19:39-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:19:39: Obtaining file (indexes:dbsnp_idx) to t +he file job store +[2020-10-10T17:19:39-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmpqv3f2j56/worker_log.txt +[2020-10-10T17:19:40-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-mhlqarxw +[2020-10-10T17:19:40-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:19:40: Obtaining file (indexes:genome_dict) to + the file job store +[2020-10-10T17:19:40-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmp2kskfmky/worker_log.txt +[2020-10-10T17:20:00-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-b4jsp5w6 +[2020-10-10T17:20:00-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:20:00: Obtaining file (transgene:gencode_annot +ation_gtf) to the file job store +[2020-10-10T17:22:12-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-1hk1njja +[2020-10-10T17:22:12-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:22:12: Obtaining file (radia:dbsnp_beds) to th +e file job store +[2020-10-10T17:29:04-0700] [MainThread] [I] [toil.leader] 1 jobs are running, +0 jobs are issued and waiting to run +[2020-10-10T17:40:50-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipe +line_inputs' kind-get_pipeline_inputs/instance-vf7ss4hd +[2020-10-10T17:40:50-0700] [MainThread] [I] [toil.leader] Issued job 'launch_p +rotect' kind-launch_protect/instance-ub2jjf37 with job batch system ID: 30 and + cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T17:40:50-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message + from job at time 10-10-2020 17:40:50: Obtaining file (star:index) to the file + job store +[2020-10-10T17:40:51-0700] [MainThread] [I] [toil.worker] Redirecting logging +to /tmp/node-419c6932-9608-4b0b-83ab-abc121f5dfa3-1e53f69f16d74e0b881e7db602b2 +d2c3/tmp65hoks_5/worker_log.txt +[2020-10-10T17:40:51-0700] [MainThread] [I] [toil.leader] Job ended: 'launch_p +rotect' kind-launch_protect/instance-ub2jjf37 + +Workflow Progress 100%|█████| 31/31 (0 failures) [1h 11:48<00:00, 0.01 jobs/s] +Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/bin/ProTECT", line 33, in + sys.exit(load_entry_point('protect', 'console_scripts', 'ProTECT')()) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1025, in main + Job.Runner.startToil(start, params) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 796, in startToil + return toil.start(job) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/common.py", line 846, in start + return self._runMainLoop(rootJobGraph) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/common.py", line 1133, in _runMainLoop + return Leader(config=self.config, + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/leader.py", line 235, in run + self.innerLoop() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/leader.py", line 558, in innerLoop + self._processReadyJobs() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/leader.py", line 482, in _processReadyJobs + self._processReadyJob(jobGraph, resultStatus) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/leader.py", line 452, in _processReadyJob + self._runJobSuccessors(jobGraph) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/leader.py", line 370, in _runJobSuccessors + self.issueJobs(successors) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/leader.py", line 697, in issueJobs + self.issueJob(job) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/leader.py", line 676, in issueJob + jobBatchSystemID = self.batchSystem.issueBatchJob(jobNode) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/batchSystems/singleMachine.py", line 497, in issueBatchJob + self.checkResourceRequest(jobNode.memory, cores, jobNode.disk, name=jobNode.jobName, + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/batchSystems/abstractBatchSystem.py", line 303, in checkResourceRequest + raise InsufficientSystemResources('disk', disk, self.maxDisk, +toil.batchSystems.abstractBatchSystem.InsufficientSystemResources: The job prepare_samples is requesting 42949672960 bytes of disk, more than the maximum of 7397748736 bytes of disk that SingleMachineBatchSystem was configured with. Scale is set to 1. diff --git a/errors/202010102013 b/errors/202010102013 new file mode 100644 index 00000000..620e10c6 --- /dev/null +++ b/errors/202010102013 @@ -0,0 +1,260 @@ +[2020-10-10T18:30:34-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmpnxpnfmic/worker_log.txt +[2020-10-10T18:30:34-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 18:30:34: Obtaining file (reports:car_t_targets_file) + to the file job store +[2020-10-10T18:42:03-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeli +ne_inputs' kind-get_pipeline_inputs/instance-3xuthybc +[2020-10-10T18:42:04-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmp9enkrl3j/worker_log.txt +[2020-10-10T18:42:04-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 18:42:04: Obtaining file (snpeff:index) to the file j +ob store +[2020-10-10T18:42:11-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 18:42:11: Obtaining file (transgene:gencode_peptide_f +asta) to the file job store +[2020-10-10T18:42:11-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeli +ne_inputs' kind-get_pipeline_inputs/instance-1tq_doef +[2020-10-10T18:42:11-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmp9xm2iz5s/worker_log.txt +[2020-10-10T18:44:05-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeli +ne_inputs' kind-get_pipeline_inputs/instance-onp40fdo +[2020-10-10T18:44:05-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 18:44:05: Obtaining file (indexes:dbsnp_vcf) to the f +ile job store +[2020-10-10T18:44:05-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmpgzfcij8r/worker_log.txt +[2020-10-10T18:44:06-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeli +ne_inputs' kind-get_pipeline_inputs/instance-bej18yp1 +[2020-10-10T18:44:06-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 18:44:06: Obtaining file (reports:mhc_pathways_file) +to the file job store +[2020-10-10T18:44:06-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmpehzwwqqa/worker_log.txt +[2020-10-10T18:44:24-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeli +ne_inputs' kind-get_pipeline_inputs/instance-68jw45m7 +[2020-10-10T18:44:24-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmpzcvq0ixe/worker_log.txt +[2020-10-10T18:44:24-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 18:44:24: Obtaining file (radia:cosmic_beds) to the f +ile job store +[2020-10-10T18:44:25-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeli +ne_inputs' kind-get_pipeline_inputs/instance-5nh8bxbe +[2020-10-10T18:44:25-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmpfezaj816/worker_log.txt +[2020-10-10T18:44:25-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 18:44:25: Obtaining file (indexes:genome_fai) to the +file job store +[2020-10-10T18:45:04-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeli +ne_inputs' kind-get_pipeline_inputs/instance-a7wxahhq +[2020-10-10T18:45:04-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmp3_s7satr/worker_log.txt +[2020-10-10T18:45:04-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 18:45:04: Obtaining file (indexes:cosmic_vcf) to the +file job store +[2020-10-10T18:45:04-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeli +ne_inputs' kind-get_pipeline_inputs/instance-8rv6x9eu +[2020-10-10T18:45:04-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmpf6qih6z0/worker_log.txt +[2020-10-10T18:45:04-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 18:45:04: Obtaining file (strelka:config_file) to the + file job store +[2020-10-10T18:45:05-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeli +ne_inputs' kind-get_pipeline_inputs/instance-1u8182hh +[2020-10-10T18:45:05-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmp_37so54e/worker_log.txt +[2020-10-10T18:45:05-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 18:45:05: Obtaining file (radia:retrogene_beds) to th +e file job store +[2020-10-10T18:45:07-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeli +ne_inputs' kind-get_pipeline_inputs/instance-1pn9ymci +[2020-10-10T18:45:07-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 18:45:07: Obtaining file (indexes:dbsnp_idx) to the f +ile job store +[2020-10-10T18:46:16-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeli +ne_inputs' kind-get_pipeline_inputs/instance-6ujlyl9w +[2020-10-10T18:46:16-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 18:46:16: Obtaining file (bwa:index) to the file job +store +[2020-10-10T19:03:24-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 19:03:24: Obtaining file (phlat:index) to the file jo +b store +[2020-10-10T19:03:24-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeli +ne_inputs' kind-get_pipeline_inputs/instance-zov6m_46 +[2020-10-10T19:10:20-0700] [MainThread] [I] [toil.leader] 1 jobs are running, 0 +jobs are issued and waiting to run +[2020-10-10T19:22:47-0700] [MainThread] [I] [toil.leader] Job ended: 'get_pipeli +ne_inputs' kind-get_pipeline_inputs/instance-g5filouy +[2020-10-10T19:22:47-0700] [MainThread] [I] [toil.leader] Issued job 'launch_pro +tect' kind-launch_protect/instance-hz1_ae6g with job batch system ID: 30 and cor +es: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-10T19:22:47-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 19:22:47: Obtaining file (star:index) to the file job + store +[2020-10-10T19:22:47-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmpxe8rlkrk/worker_log.txt +[2020-10-10T19:22:48-0700] [MainThread] [I] [toil.leader] Job ended: 'launch_pro +tect' kind-launch_protect/instance-hz1_ae6g +[2020-10-10T19:22:48-0700] [MainThread] [I] [toil.leader] Issued job 'prepare_sa +mples' kind-prepare_samples/instance-ov48vrxg with job batch system ID: 31 and c +ores: 1, disk: 40.0 G, and memory: 2.0 G +[2020-10-10T19:22:49-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmpou9vug5p/worker_log.txt +[2020-10-10T19:23:31-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 19:23:31: Downloading Inputs for TEST +[2020-10-10T19:23:31-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 19:23:31: Obtaining file (TEST:normal_dna_fastq_2) to + the file job store +[2020-10-10T19:23:31-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 19:23:31: Obtaining file (TEST:tumor_rna_fastq_1) to +the file job store +[2020-10-10T19:23:31-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 19:23:31: Obtaining file (TEST:tumor_dna_fastq_1) to +the file job store +[2020-10-10T19:23:31-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 19:23:31: Obtaining file (TEST:normal_dna_fastq_1) to + the file job store +[2020-10-10T19:23:31-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 19:23:31: Obtaining file (TEST:tumor_rna_fastq_2) to +the file job store +[2020-10-10T19:23:31-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message f +rom job at time 10-10-2020 19:23:31: Obtaining file (TEST:tumor_dna_fastq_2) to +the file job store +[2020-10-10T19:23:31-0700] [MainThread] [I] [toil.leader] Job ended: 'prepare_sa +mples' kind-prepare_samples/instance-ov48vrxg +[2020-10-10T19:23:31-0700] [MainThread] [I] [toil.leader] Issued job 'get_patien +t_fastqs' kind-get_patient_fastqs/instance-uebtou6z with job batch system ID: 32 + and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-10-10T19:23:31-0700] [MainThread] [I] [toil.leader] Issued job 'get_patien +t_fastqs' kind-get_patient_fastqs/instance-vid4ipgt with job batch system ID: 33 + and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-10-10T19:23:31-0700] [MainThread] [I] [toil.leader] Issued job 'get_patien +t_fastqs' kind-get_patient_fastqs/instance-thvdi_2_ with job batch system ID: 34 + and cores: 1, disk: 10.0 M, and memory: 2.0 G +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmpz31v839a/worker_log.txt +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmpf8wh3lv1/worker_log.txt +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmp55svqdsp/worker_log.txt +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.leader] Job ended: 'get_patien +t_fastqs' kind-get_patient_fastqs/instance-thvdi_2_ +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.leader] Issued job 'Encapsulat +edJob' kind-EncapsulatedJob/instance-pf4569cs with job batch system ID: 35 and c +ores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.leader] Issued job 'Encapsulat +edJob' kind-EncapsulatedJob/instance-wrmal69c with job batch system ID: 36 and c +ores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.leader] Job ended: 'get_patien +t_fastqs' kind-get_patient_fastqs/instance-vid4ipgt +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.leader] Issued job 'Encapsulat +edJob' kind-EncapsulatedJob/instance-axz3wj0j with job batch system ID: 37 and c +ores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.leader] Issued job 'Encapsulat +edJob' kind-EncapsulatedJob/instance-lu8fhwd_ with job batch system ID: 38 and c +ores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.leader] Job ended: 'get_patien +t_fastqs' kind-get_patient_fastqs/instance-uebtou6z +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.leader] Issued job 'Encapsulat +edJob' kind-EncapsulatedJob/instance-idyv9uyv with job batch system ID: 39 and c +ores: 0, disk: 1.0 M, and memory: 32.0 M +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.leader] Issued job 'Encapsulat +edJob' kind-EncapsulatedJob/instance-_vz1u_x7 with job batch system ID: 40 and c +ores: 1, disk: 100.0 M, and memory: 2.0 G +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmpbixrdmpk/worker_log.txt +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmpjihwvv70/worker_log.txt +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmpnjzrq_r2/worker_log.txt +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.leader] Job ended: 'Encapsulat +edJob' kind-EncapsulatedJob/instance-pf4569cs +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmpflulx9f5/worker_log.txt +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' + kind-run_phlat/instance-ft5phwm5 with job batch system ID: 41 and cores: 4, dis +k: 7.7 G, and memory: 2.0 G +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmpady15hlr/worker_log.txt +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.leader] Job ended: 'Encapsulat +edJob' kind-EncapsulatedJob/instance-wrmal69c +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.leader] Issued job 'run_cutada +pt' kind-run_cutadapt/instance-6f8cn5ks with job batch system ID: 42 and cores: +1, disk: 539.2 M, and memory: 2.0 G +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmp2l_dsop_/worker_log.txt +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.leader] Job ended: 'Encapsulat +edJob' kind-EncapsulatedJob/instance-axz3wj0j +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' + kind-run_phlat/instance-k06mp0_z with job batch system ID: 43 and cores: 4, dis +k: 7.7 G, and memory: 2.0 G +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.leader] Job ended: 'Encapsulat +edJob' kind-EncapsulatedJob/instance-idyv9uyv +[2020-10-10T19:23:32-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' + kind-run_phlat/instance-wqt7mcyf with job batch system ID: 44 and cores: 4, dis +k: 7.8 G, and memory: 2.0 G +[2020-10-10T19:23:33-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmpcihfmlee/worker_log.txt +[2020-10-10T19:23:33-0700] [MainThread] [I] [toil.leader] Job ended: 'Encapsulat +edJob' kind-EncapsulatedJob/instance-lu8fhwd_ +[2020-10-10T19:23:33-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Go +t exit code 1 (indicating failure) from job _toil_worker run_cutadapt file:/home +/drkthomp/protect3/jobStore kind-run_cutadapt/instance-6f8cn5ks. +[2020-10-10T19:23:33-0700] [MainThread] [I] [toil.worker] Redirecting logging to + /home/drkthomp/workDir/node-cbabd37a-13ea-4540-bb67-748b76bc85e7-1e53f69f16d74e +0b881e7db602b2d2c3/tmpjcpzwd4a/worker_log.txt + +Workflow Progress 89%|██████▎| 40/45 (0 failures) [1h 13:14<09:09, 0.01 jobs/s] +Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/bin/ProTECT", line 33, in + sys.exit(load_entry_point('protect', 'console_scripts', 'ProTECT')()) + File "/home/drkthomp/protect3/src/protect/pipeline/ProTECT.py", line 1025, in main + Job.Runner.startToil(start, params) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 796, in startToil + return toil.start(job) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/common.py", line 846, in start + return self._runMainLoop(rootJobGraph) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/common.py", line 1133, in _runMainLoop + return Leader(config=self.config, + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/leader.py", line 235, in run + self.innerLoop() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/leader.py", line 558, in innerLoop + self._processReadyJobs() + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/leader.py", line 482, in _processReadyJobs + self._processReadyJob(jobGraph, resultStatus) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/leader.py", line 452, in _processReadyJob + self._runJobSuccessors(jobGraph) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/leader.py", line 370, in _runJobSuccessors + self.issueJobs(successors) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/leader.py", line 697, in issueJobs + self.issueJob(job) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/leader.py", line 676, in issueJob + jobBatchSystemID = self.batchSystem.issueBatchJob(jobNode) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/batchSystems/singleMachine.py", line 497, in issueBatchJob + self.checkResourceRequest(jobNode.memory, cores, jobNode.disk, name=jobNode.jobName, + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/batchSystems/abstractBatchSystem.py", line 300, in checkResourceRequest + raise InsufficientSystemResources('memory', memory, self.maxMemory, +toil.batchSystems.abstractBatchSystem.InsufficientSystemResources: The job run_bwa is requesting 21474836480 bytes of memory, more than the maximum of 14712700928 bytes of memory that SingleMachineBatchSystem was configured with. Scale is set to 1. + diff --git a/errors/202010110258 phlat b/errors/202010110258 phlat new file mode 100644 index 00000000..d4cd021d --- /dev/null +++ b/errors/202010110258 phlat @@ -0,0 +1,355 @@ +[2020-10-11T02:45:49-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-vrhfvnhs +[2020-10-11T02:45:49-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-8xpgm6ib +[2020-10-11T02:45:49-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_cutadapt/instance-xfrh8ah4 +[2020-10-11T02:45:49-0700] [MainThread] [C] [toil.jobStores.abstractJobStore] Repairing job: kind-run_phlat/instance-a5ark53j +[2020-10-11T02:45:49-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. +[2020-10-11T02:45:49-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instanc +e-vrhfvnhs with job batch system ID: 0 and cores: 4, disk: 7.8 G, and memory: 2.0 G +[2020-10-11T02:45:49-0700] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-r5 +xtr9zr with job batch system ID: 1 and cores: 4, disk: 9.6 G, and memory: 20.0 G +[2020-10-11T02:45:49-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instanc +e-8xpgm6ib with job batch system ID: 2 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-11T02:45:49-0700] [MainThread] [I] [toil.leader] Issued job 'run_bwa' kind-run_bwa/instance-h_ +u0bx5b with job batch system ID: 3 and cores: 4, disk: 9.4 G, and memory: 20.0 G +[2020-10-11T02:45:49-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instanc +e-a5ark53j with job batch system ID: 4 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-11T02:45:49-0700] [MainThread] [I] [toil.leader] Issued job 'run_cutadapt' kind-run_cutadapt/i +nstance-xfrh8ah4 with job batch system ID: 5 and cores: 1, disk: 2.0 G, and memory: 2.0 G +[2020-10-11T02:45:49-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir +/node-63724b0a-38c3-4733-a482-9b7c3ca99788-1e53f69f16d74e0b881e7db602b2d2c3/tmpzmy3wdo7/worker_log.txt +[2020-10-11T02:45:51-0700] [MainThread] [I] [toil.leader] 1 jobs are running, 5 jobs are issued and wai +ting to run +[2020-10-11T02:46:35-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicati +ng failure) from job _toil_worker run_phlat file:/home/drkthomp/protect3/jobStore kind-run_phlat/instan +ce-vrhfvnhs. +[2020-10-11T02:46:35-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kin +d-run_phlat/instance-vrhfvnhs +[2020-10-11T02:46:35-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indica +ting failure: 'run_phlat' kind-run_phlat/instance-vrhfvnhs +[2020-10-11T02:46:35-0700] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-vrhfvnhs + follows: +=========> + [2020-10-11T02:45:49-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-11T02:45:49-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb +357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + Error response from daemon: Get "https://registry-1.docker.io/v2/": net/http: TLS handshake tim +eout + Traceback (most recent call last): + File "/home/drkthomp/protect3/src/protect/common.py", line 120, in docker_call + subprocess.check_call(call) + File "/usr/lib/python3.8/subprocess.py", line 364, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'pull', 'aarjunrao/phlat:1.0']' returned non +-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in +workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _r +unner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _r +un + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in ru +n + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 72, in run_phlat + docker_call(tool='phlat', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 122, in docker_call + raise RuntimeError('docker command returned a non-zero exit status ' + + RuntimeError: docker command returned a non-zero exit status (1)for command "docker pull aarjun +rao/phlat:1.0" + [2020-10-11T02:46:34-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a faile +d job on host drkthomp-virtualbox +<========= +[2020-10-11T02:46:35-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remainin +g retry count of job 'run_phlat' kind-run_phlat/instance-vrhfvnhs with ID kind-run_phlat/instance-vrhfv +nhs to 1 +[2020-10-11T02:46:35-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instanc +e-vrhfvnhs with job batch system ID: 6 and cores: 4, disk: 7.8 G, and memory: 2.0 G +[2020-10-11T02:46:35-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir +/node-63724b0a-38c3-4733-a482-9b7c3ca99788-1e53f69f16d74e0b881e7db602b2d2c3/tmpl8w7g8he/worker_log.txt +[2020-10-11T02:47:36-0700] [MainThread] [I] [toil.leader] Job ended: 'run_cutadapt' kind-run_cutadapt/i +nstance-xfrh8ah4 +[2020-10-11T02:47:36-0700] [MainThread] [I] [toil.leader] Issued job 'Job' kind-Job/instance-gietgg6t w +ith job batch system ID: 7 and cores: 0, disk: 100.0 M, and memory: 512.0 M +[2020-10-11T02:47:36-0700] [Thread-3 ] [I] [toil.statsAndLogging] Got message from job at time 10-11-2 +020 02:47:36: Ran cutadapt on TEST successfully +[2020-10-11T02:47:36-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir +/node-63724b0a-38c3-4733-a482-9b7c3ca99788-1e53f69f16d74e0b881e7db602b2d2c3/tmpcstkpeka/worker_log.txt +[2020-10-11T02:50:24-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicati +ng failure) from job _toil_worker run_phlat file:/home/drkthomp/protect3/jobStore kind-run_phlat/instan +ce-8xpgm6ib. +[2020-10-11T02:50:24-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kin +d-run_phlat/instance-8xpgm6ib +[2020-10-11T02:50:24-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indica +ting failure: 'run_phlat' kind-run_phlat/instance-8xpgm6ib +[2020-10-11T02:50:24-0700] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-8xpgm6ib + follows: +=========> + [2020-10-11T02:47:36-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-11T02:47:36-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb +357efe36d7df9b9f6ae9b5b on host drkthomp-virtualbox. + 1.0: Pulling from aarjunrao/phlat + 943c334059c7: Already exists + a1acf99303d2: Already exists + 27616aacb7b3: Already exists + 35d12cd1c9fc: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + a3ed95caeb02: Already exists + a27d4604b3b3: Already exists + b23184322fd8: Already exists + 45490e2c3ebf: Already exists + a3ed95caeb02: Already exists + 07bc45a5d3f9: Already exists + a3ed95caeb02: Already exists + 4e8282f9b0b2: Pulling fs layer + d8467ea2e124: Pulling fs layer + b53fc7d0498f: Pulling fs layer + 30ab0013e0b7: Pulling fs layer + 949d3b3c935b: Pulling fs layer + a98c227726f6: Pulling fs layer + 4347594c2d1a: Pulling fs layer + f13886bcc594: Pulling fs layer + a3ed95caeb02: Pulling fs layer + 4347594c2d1a: Waiting + f13886bcc594: Waiting + a3ed95caeb02: Waiting + 949d3b3c935b: Waiting + a98c227726f6: Waiting + 30ab0013e0b7: Waiting + 4e8282f9b0b2: Verifying Checksum + 4e8282f9b0b2: Download complete + 4e8282f9b0b2: Pull complete + b53fc7d0498f: Verifying Checksum + b53fc7d0498f: Download complete + d8467ea2e124: Verifying Checksum + d8467ea2e124: Download complete + 30ab0013e0b7: Verifying Checksum + 30ab0013e0b7: Download complete + a98c227726f6: Verifying Checksum + a98c227726f6: Download complete + f13886bcc594: Verifying Checksum + f13886bcc594: Download complete + d8467ea2e124: Pull complete + 949d3b3c935b: Verifying Checksum + 949d3b3c935b: Download complete + a3ed95caeb02: Verifying Checksum + a3ed95caeb02: Download complete + 4347594c2d1a: Verifying Checksum + 4347594c2d1a: Download complete + b53fc7d0498f: Pull complete + 30ab0013e0b7: Pull complete + 949d3b3c935b: Pull complete + a98c227726f6: Pull complete + 4347594c2d1a: Pull complete + f13886bcc594: Pull complete + a3ed95caeb02: Pull complete + Digest: sha256:a11b92ed62d9e189ae36823abeb405c8bca9f41f47fd0d956397c539f8534b5f + Status: Downloaded newer image for aarjunrao/phlat:1.0 + docker.io/aarjunrao/phlat:1.0 + 491247 reads; of these: + 491247 (100.00%) were paired; of these: + 3488 (0.71%) aligned concordantly 0 times + 442898 (90.16%) aligned concordantly exactly 1 time + 44861 (9.13%) aligned concordantly >1 times + 99.29% overall alignment rate + .....Process Bowtie 2 mapping on normal_dna....... + + .....Prepare files of normal_dna for PHLAT....... + + ..... Running PHLAT ....... + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 64, in + go(tag,outdir,phlatdir) + File "/home/yubai/programs/phlat-1.0/dist/utilities.py", line 79, in go + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 617, in __init__ + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 645, in setMaster + IndexError: list index out of range + Traceback (most recent call last): + File "/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib/python3.8/subprocess.py", line 364, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/home/drkthomp/wo +rkDir/node-63724b0a-38c3-4733-a482-9b7c3ca99788-1e53f69f16d74e0b881e7db602b2d2c3/tmpcstkpeka/e48ad10f-5 +c07-4855-b05f-65eacef49a00:/data', '--log-driver=none', 'aarjunrao/phlat:1.0', '-1', '/data/input_1.fas +tq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2', '- +tag', 'normal_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '4']' returned non-zero exit status 1 +. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in +workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _r +unner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _r +un + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in ru +n + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 72, in run_phlat + docker_call(tool='phlat', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 138, in docker_call + raise RuntimeError('docker command returned a non-zero exit status (%s)' % err.returncode + + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=tr +ue -v /home/drkthomp/workDir/node-63724b0a-38c3-4733-a482-9b7c3ca99788-1e53f69f16d74e0b881e7db602b2d2c3 +/tmpcstkpeka/e48ad10f-5c07-4855-b05f-65eacef49a00:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/ +input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag normal +_dna -e /home/phlat-1.0 -o /data -p 4" + [2020-10-11T02:50:24-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a faile +d job on host drkthomp-virtualbox +<========= +[2020-10-11T02:50:24-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remainin +g retry count of job 'run_phlat' kind-run_phlat/instance-8xpgm6ib with ID kind-run_phlat/instance-8xpgm +6ib to 1 +[2020-10-11T02:50:24-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instanc +e-8xpgm6ib with job batch system ID: 8 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-11T02:50:25-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir +/node-63724b0a-38c3-4733-a482-9b7c3ca99788-1e53f69f16d74e0b881e7db602b2d2c3/tmpvppnj5ub/worker_log.txt +[2020-10-11T02:52:26-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker + run_phlat file:/home/drkthomp/protect3/jobStore kind-run_phlat/instance-a5ark53j. +[2020-10-11T02:52:26-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-a5ark53j +[2020-10-11T02:52:26-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-ru +n_phlat/instance-a5ark53j +[2020-10-11T02:52:26-0700] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-a5ark53j follows: +=========> + [2020-10-11T02:50:25-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-11T02:50:25-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host d +rkthomp-virtualbox. + 462668 reads; of these: + 462668 (100.00%) were paired; of these: + 75207 (16.26%) aligned concordantly 0 times + 235430 (50.89%) aligned concordantly exactly 1 time + 152031 (32.86%) aligned concordantly >1 times + 83.74% overall alignment rate + .....Process Bowtie 2 mapping on tumor_rna....... + + .....Prepare files of tumor_rna for PHLAT....... + + ..... Running PHLAT ....... + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 64, in + go(tag,outdir,phlatdir) + File "/home/yubai/programs/phlat-1.0/dist/utilities.py", line 79, in go + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 617, in __init__ + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 645, in setMaster + IndexError: list index out of range + Traceback (most recent call last): + File "/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib/python3.8/subprocess.py", line 364, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/home/drkthomp/workDir/node-63724b0a-38c3-4733-a48 +2-9b7c3ca99788-1e53f69f16d74e0b881e7db602b2d2c3/tmpvppnj5ub/6824d77d-283e-4c5a-9cc5-c77bf0460ae9:/data', '--log-driver=none', 'aarjunrao +/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2' +, '-tag', 'tumor_rna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '4']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 72, in run_phlat + docker_call(tool='phlat', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 138, in docker_call + raise RuntimeError('docker command returned a non-zero exit status (%s)' % err.returncode + + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /home/drkthomp/workDir/node +-63724b0a-38c3-4733-a482-9b7c3ca99788-1e53f69f16d74e0b881e7db602b2d2c3/tmpvppnj5ub/6824d77d-283e-4c5a-9cc5-c77bf0460ae9:/data --log-driv +er=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag tu +mor_rna -e /home/phlat-1.0 -o /data -p 4" + [2020-10-11T02:52:26-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-11T02:52:26-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' +kind-run_phlat/instance-a5ark53j with ID kind-run_phlat/instance-a5ark53j to 1 +[2020-10-11T02:52:26-0700] [MainThread] [I] [toil.leader] Issued job 'run_phlat' kind-run_phlat/instance-a5ark53j with job batch system +ID: 9 and cores: 4, disk: 7.7 G, and memory: 2.0 G +[2020-10-11T02:52:26-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-63724b0a-38c3-4733-a482-9b7 +c3ca99788-1e53f69f16d74e0b881e7db602b2d2c3/tmpqvjh0h0n/worker_log.txt +[2020-10-11T02:55:15-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_worker + run_phlat file:/home/drkthomp/protect3/jobStore kind-run_phlat/instance-8xpgm6ib. +[2020-10-11T02:55:15-0700] [MainThread] [W] [toil.leader] Job failed with exit value 1: 'run_phlat' kind-run_phlat/instance-8xpgm6ib +[2020-10-11T02:55:15-0700] [MainThread] [W] [toil.leader] The job seems to have left a log file, indicating failure: 'run_phlat' kind-ru +n_phlat/instance-8xpgm6ib +[2020-10-11T02:55:15-0700] [MainThread] [W] [toil.leader] Log from job kind-run_phlat/instance-8xpgm6ib follows: +=========> + [2020-10-11T02:52:26-0700] [MainThread] [I] [toil.worker] ---TOIL WORKER OUTPUT LOG--- + [2020-10-11T02:52:26-0700] [MainThread] [I] [toil] Running Toil version 4.2.0-3aa1da130141039cb357efe36d7df9b9f6ae9b5b on host d +rkthomp-virtualbox. + 491247 reads; of these: + 491247 (100.00%) were paired; of these: + 3488 (0.71%) aligned concordantly 0 times + 442898 (90.16%) aligned concordantly exactly 1 time + 44861 (9.13%) aligned concordantly >1 times + 99.29% overall alignment rate + .....Process Bowtie 2 mapping on normal_dna....... + + .....Prepare files of normal_dna for PHLAT....... + + ..... Running PHLAT ....... + Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 64, in + go(tag,outdir,phlatdir) + File "/home/yubai/programs/phlat-1.0/dist/utilities.py", line 79, in go + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 617, in __init__ + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 645, in setMaster + IndexError: list index out of range + Traceback (most recent call last): + File "/home/drkthomp/protect3/src/protect/common.py", line 136, in docker_call + subprocess.check_call(call, stdout=outfile) + File "/usr/lib/python3.8/subprocess.py", line 364, in check_call + raise CalledProcessError(retcode, cmd) + subprocess.CalledProcessError: Command '['docker', 'run', '--rm=true', '-v', '/home/drkthomp/workDir/node-63724b0a-38c3-4733-a48 +2-9b7c3ca99788-1e53f69f16d74e0b881e7db602b2d2c3/tmpqvjh0h0n/06bb4ff5-ef0a-4186-8805-9d2d424211c8:/data', '--log-driver=none', 'aarjunrao +/phlat:1.0', '-1', '/data/input_1.fastq', '-2', '/data/input_2.fastq', '-index', '/data/index4phlat', '-b2url', '/usr/local/bin/bowtie2' +, '-tag', 'normal_dna', '-e', '/home/phlat-1.0', '-o', '/data', '-p', '4']' returned non-zero exit status 1. + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/worker.py", line 368, in workerScript + job._runner(jobGraph=jobGraph, jobStore=jobStore, fileStore=fileStore, defer=defer) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1424, in _runner + returnValues = self._run(jobGraph, fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1361, in _run + return self.run(fileStore) + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/job.py", line 1565, in run + rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs) + File "/home/drkthomp/protect3/src/protect/haplotyping/phlat.py", line 72, in run_phlat + docker_call(tool='phlat', tool_parameters=parameters, work_dir=work_dir, + File "/home/drkthomp/protect3/src/protect/common.py", line 138, in docker_call + raise RuntimeError('docker command returned a non-zero exit status (%s)' % err.returncode + + RuntimeError: docker command returned a non-zero exit status (1)for command "docker run --rm=true -v /home/drkthomp/workDir/node +-63724b0a-38c3-4733-a482-9b7c3ca99788-1e53f69f16d74e0b881e7db602b2d2c3/tmpqvjh0h0n/06bb4ff5-ef0a-4186-8805-9d2d424211c8:/data --log-driv +er=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag no +rmal_dna -e /home/phlat-1.0 -o /data -p 4" + [2020-10-11T02:55:15-0700] [MainThread] [E] [toil.worker] Exiting the worker because of a failed job on host drkthomp-virtualbox +<========= +[2020-10-11T02:55:15-0700] [MainThread] [W] [toil.jobGraph] Due to failure we are reducing the remaining retry count of job 'run_phlat' +kind-run_phlat/instance-8xpgm6ib with ID kind-run_phlat/instance-8xpgm6ib to 0 +[2020-10-11T02:55:15-0700] [MainThread] [W] [toil.leader] Job 'run_phlat' kind-run_phlat/instance-8xpgm6ib with ID kind-run_phlat/instan +ce-8xpgm6ib is completely failed +[2020-10-11T02:55:15-0700] [MainThread] [I] [toil.worker] Redirecting logging to /home/drkthomp/workDir/node-63724b0a-38c3-4733-a482-9b7 +c3ca99788-1e53f69f16d74e0b881e7db602b2d2c3/tmpmtp81cj4/worker_log.txt +^C^C[2020-10-11T02:56:18-0700] [Thread-1 ] [E] [toil.batchSystems.singleMachine] Got exit code 1 (indicating failure) from job _toil_wo +rker run_phlat file:/home/drkthomp/protect3/jobStore kind-run_phlat/instance-vrhfvnhs. +^CTraceback (most recent call last): + File "/home/drkthomp/protect3/venv/bin/_toil_worker", line 5, in + from toil.worker import main + File "/home/drkthomp/protect3/venv/lib/python3.8/site-packages/toil/__init__.py", line 20, in diff --git a/errors/20201019 actualerror b/errors/20201019 actualerror new file mode 100644 index 00000000..35959e76 --- /dev/null +++ b/errors/20201019 actualerror @@ -0,0 +1,18 @@ +7331b3-1e53f69f16d74e0b881e7db602b2d2c3/tmpbf77rqv6/57fad350-313b-40b4-b8b7-103f860efde4:/data --log-driver=none aarjunrao/phlat:1.0 -1 /data/input_1.fastq -2 /data/input_2.fastq -index /data/index4phlat -b2url /usr/local/bin/bowtie2 -tag normal_dna -e /home/phlat-1.0 -o /data -p 4 +Could not locate a Bowtie index corresponding to basename "/data/index4phlat/ucsc.artHLA" +Error: Encountered internal Bowtie 2 exception (#1) +Command: /usr/local/bin/bowtie2-align-s --wrapper basic-0 --maxins 450 --no-mixed --no-discordant -p 4 -D 20 -R 3 -N 0 -L 20 -i S,1,0.50 -x /data/index4phlat/ucsc.artHLA --passthrough -1 /data/input_1.fastq -2 /data/input_2.fastq +(ERR): bowtie2-align exited with value 1 +.....Process Bowtie 2 mapping on normal_dna....... + +.....Prepare files of normal_dna for PHLAT....... + +..... Running PHLAT ....... +Traceback (most recent call last): + File "/home/phlat-1.0/dist/PHLAT.py", line 64, in + go(tag,outdir,phlatdir) + File "/home/yubai/programs/phlat-1.0/dist/utilities.py", line 79, in go + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 617, in __init__ + File "/home/yubai/programs/phlat-1.0/dist/extensions.py", line 645, in setMaster +IndexError: list index out of range + diff --git a/mustard_config.yaml b/mustard_config.yaml new file mode 100755 index 00000000..cd21dbbf --- /dev/null +++ b/mustard_config.yaml @@ -0,0 +1,179 @@ +## Copyright 2016 UCSC Computational Genomics Lab +## Original contributor: Arjun Arkal Rao +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +#################################################################################################### +#################################################################################################### +## This is the input parameter file for the precision immuno pipeline. The parameters for each of +## the tools is provided here. The file is written in the YAML format. A nice description of the +## format can be found at http://docs.ansible.com/ansible/YAMLSyntax.html +## +## You can add comments anywhere in this file by prefixing it with a '#' +## +## Unless otherwise mentioned, all fields must be filled. +## +#################################################################################################### +#################################################################################################### + +# Any number of patients/samples can be listed here +patients: + TEST: + tumor_dna_fastq_1: /scratch/drkthomp/gerald_D1VCPACXX_1_1.fastq + tumor_dna_fastq_2: /scratch/drkthomp/gerald_D1VCPACXX_1_2.fastq + normal_dna_fastq_1: /scratch/drkthomp/gerald_D1VCPACXX_6_1.fastq + normal_dna_fastq_2: /scratch/drkthomp/gerald_D1VCPACXX_6_2.fastq + tumor_rna_fastq_1: /scratch/drkthomp/gerald_C1TD1ACXX_8_ACAGTG_1.fastq + tumor_rna_fastq_2: /scratch/drkthomp/gerald_C1TD1ACXX_8_ACAGTG_2.fastq + tumor_type: 'SKCM' + +# These are options that are used by most tools +Universal_Options: + dockerhub: aarjunrao + java_Xmx: 12G + reference_build: hg38 # Acceptable options are hg38, hg38, GRCh37, GRCh38 + # sse_key: /path/to/master.key # Path to the AWS master key. Required if using AWS else optional + # sse_key_is_master: True # True or False. Required if using AWS else optional + # gdc_download_token: /path/to/token.txt # Path to the user's GDC download token. + storage_location: Local # Local or aws: for where the output must go + #storage_location: aws:protect-run-xyz + output_folder: /private/home/drkthomp/results # Path to where the output must go. + #mail_to: test.email@host.com # Email patients: + #TEST: + # tumor_dna_fastq_1: /home/drkthomp/e/treehouse/gms/hcc1395_1tenth_percent/sort/gerald_D1VCPACXX_1_R1.fastq + # tumor_dna_fastq_2: /home/drkthomp/e/treehouse/gms/hcc1395_1tenth_percent/sort/gerald_D1VCPACXX_1_R2.fastq + # normal_dna_fastq_1: /home/drkthomp/e/treehouse/gms/hcc1395_1tenth_percent/sort/gerald_D1VCPACXX_6_R1.fastq + # normal_dna_fastq_2: /home/drkthomp/e/treehouse/gms/hcc1395_1tenth_percent/sort/gerald_D1VCPACXX_6_R2.fastq + # tumor_rna_fastq_1: /home/drkthomp/e/treehouse/gms/hcc1395_1tenth_percent/sort/gerald_C1TD1ACXX_8_ACAGTG_R1.fastq + # tumor_rna_fastq_2: /home/drkthomp/e/treehouse/gms/hcc1395_1tenth_percent/sort/gerald_C1TD1ACXX_8_ACAGTG_R2.fastq + # tumor_type: 'SKCM' + +# for sending success report. + + +# These options are for each module. You probably don't need to change any of this! +alignment: + cutadapt: + a: AGATCGGAAGAG + A: AGATCGGAAGAG + # version: 1.9.1 + star: + type: star # use starlong if your reads are > 150bp + index: /scratch/drkthomp/protect-index/star_with_fusion_100bp_readlen_indexes.tar.gz # Use star_without if you set star_fusion = False + # version: 2.5.2b + bwa: + index: /scratch/drkthomp/protect-index/bwa_index.tar.gz + # version: 0.7.9a + post: + samtools: + # version: 1.2 + picard: + # version: 1.135 + +expression_estimation: + rsem: + index: /scratch/drkthomp/protect-index/rsem_index.tar.gz + # version: 1.2.0 + +mutation_calling: + indexes: + chromosomes: canonical_chr, chrM + genome_fasta: /scratch/drkthomp/protect-index/hg38.fa.tar.gz + genome_fai: /scratch/drkthomp/protect-index/hg38.fa.fai.tar.gz + genome_dict: /scratch/drkthomp/protect-index/hg38.dict.tar.gz + cosmic_vcf: /scratch/drkthomp/protect-index/CosmicCodingMuts.vcf.tar.gz + cosmic_idx: /scratch/drkthomp/protect-index/CosmicCodingMuts.vcf.idx.tar.gz + dbsnp_vcf: /scratch/drkthomp/protect-index/dbsnp_coding.vcf.gz + dbsnp_idx: /scratch/drkthomp/protect-index/dbsnp_coding.vcf.idx.tar.gz + dbsnp_tbi: /scratch/drkthomp/protect-index/dbsnp_coding.vcf.gz.tbi + mutect: + java_Xmx: 2G + # version: 1.1.7 + muse: + # version: 1.0rc_submission_b391201 + radia: + cosmic_beds: /scratch/drkthomp/protect-index/radia_cosmic.tar.gz + dbsnp_beds: /scratch/drkthomp/protect-index/radia_dbsnp.tar.gz + retrogene_beds: /scratch/drkthomp/protect-index/radia_retrogenes.tar.gz + pseudogene_beds: /scratch/drkthomp/protect-index/radia_pseudogenes.tar.gz + gencode_beds: /scratch/drkthomp/protect-index/radia_gencode.tar.gz + # version: 398366ef07b5911d8082ed61cbf03d487a41f286 + somaticsniper: + # version: 1.0.4 + samtools: + # version: 0.1.8 + bam_readcount: + # version: 0.7.4 + star_fusion: + #run: True + #version: 1.0.0 + fusion_inspector: + #run_trinity: True + #version: 1.0.1 + strelka: + # version: 1.0.15 + config_file: /scratch/drkthomp/protect-index/strelka_bwa_WXS_config.ini.tar.gz + + +mutation_annotation: + snpeff: + index: /scratch/drkthomp/protect-index/snpeff_index.tar.gz + # version: 3.6 + java_Xmx: 20G + +mutation_translation: + transgene: + gencode_peptide_fasta : /scratch/drkthomp/protect-index/gencode.v25.pc_translations_NOPARY.fa.tar.gz + gencode_transcript_fasta : /scratch/drkthomp/protect-index/gencode.v25.pc_transcripts_NOPARY.fa.tar.gz + gencode_annotation_gtf : /scratch/drkthomp/protect-index/gencode.v25.annotation_NOPARY.gtf.tar.gz + genome_fasta : /scratch/drkthomp/protect-index/hg38.fa.tar.gz + # version: 2.2.2 + +haplotyping: + phlat: + index: /scratch/drkthomp/protect-index/phlat_index.tar.gz + # version: 1.0 + +mhc_peptide_binding: + mhci: + method_file: /scratch/drkthomp/protect-index/mhci_restrictions.json.tar.gz + pred: IEDB_recommended + # version: 2.13 + mhcii: + method_file: /scratch/drkthomp/protect-index/mhcii_restrictions.json.tar.gz + pred: IEDB_recommended + # version: 2.13 + netmhciipan: + # version: 3.1 + +prediction_ranking: + rankboost: + mhci_args: + npa: 0.0 + nph: 0.0 + nMHC: 0.32 + TPM: 0.0 + overlap: 0.68 + tndelta: 0.0 + mhcii_args: + npa: 0.2 + nph: 0.2 + nMHC: 0.2 + TPM: 0.2 + tndelta: 0.2 + # version: 2.0.3 + +reports: + mhc_pathways_file: /scratch/drkthomp/protect-index/mhc_pathways.tsv.tar.gz + itx_resistance_file: /scratch/drkthomp/protect-index/itx_resistance.tsv.tar.gz + immune_resistance_pathways_file: /scratch/drkthomp/protect-index/immune_resistance_pathways.json.tar.gz + car_t_targets_file: /scratch/drkthomp/protect-index/car_t_targets.tsv.tar.gz diff --git a/run.sh b/run.sh new file mode 100755 index 00000000..b4fe4450 --- /dev/null +++ b/run.sh @@ -0,0 +1,3 @@ +source venv/bin/activate +sudo ProTECT --config ProTECT_config.yaml --workDir /home/drkthomp/workDir /home/drkthomp/d/jobStore --restart |& tee errors/$(date '+%Y-%m-%d-%H-%M-%S').txt + diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 96cb4ebe..03c4a4b0 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -14,25 +14,25 @@ # See the License for the specific language governing permissions and # limitations under the License. from pkg_resources import parse_version -try: - from pkg_resources import SetuptoolsLegacyVersion as _LegacyVersion -except ImportError as e: - if 'SetuptoolsLegacyVersion' in e.message: - from packaging.version import LegacyVersion as _LegacyVersion - else: - raise +#try: +# from pkg_resources import SetuptoolsLegacyVersion as _LegacyVersion +#except ImportError as e: +# if 'SetuptoolsLegacyVersion' in e.message: +# from packaging.version import LegacyVersion as _LegacyVersion +# else: +# raise from setuptools import find_packages, setup from setuptools.command.test import test as TestCommand -from version import version +#from version import version import errno import subprocess import sys - -toil_version = '3.8.0' -s3am_version = '2.0.1' -gdc_version = 'v1.1.0' +#outdated for python3 +#toil_version = '3.8.0' +#s3am_version = '2.0.1' +#gdc_version = 'v1.1.0' def check_tool_version(tool, required_version, blacklisted_versions=None, binary=False): @@ -66,9 +66,9 @@ def check_tool_version(tool, required_version, blacklisted_versions=None, binary raise RuntimeError('Does %s have a version.py?' % tool) if type(parse_version(installed_version)) == _LegacyVersion: - print('Detecting that the installed version of "%s"(%s) is probably based off a git commit ' + print(('Detecting that the installed version of "%s"(%s) is probably based off a git commit ' 'and assuming this build is for testing purposes. If this is not the case, please ' - 'try again with a valid version of "%s".' % (tool, installed_version, tool)) + 'try again with a valid version of "%s".' % (tool, installed_version, tool))) elif parse_version(installed_version) < parse_version(required_version): raise RuntimeError('%s was detected to be version (%s) but ProTECT requires (%s)' % (tool, installed_version, required_version)) @@ -79,11 +79,11 @@ def check_tool_version(tool, required_version, blacklisted_versions=None, binary # Check Toil version -check_tool_version('toil', toil_version, binary=True) +#check_tool_version('toil', toil_version, binary=True) # Check S3am version -check_tool_version('s3am', s3am_version, binary=True) +#check_tool_version('s3am', s3am_version, binary=True) # Check gdc-client version -check_tool_version('gdc-client', gdc_version, binary=True, blacklisted_versions=['v1.2.0']) +#check_tool_version('gdc-client', gdc_version, binary=True, blacklisted_versions=['v1.2.0']) # Set up a test class @@ -109,7 +109,6 @@ def run_tests(self): setup(name='protect', - version=version, description='Prediction of T-Cell Epitopes for Cancer Therapy', url='http://github.com/BD2KGenomics/protect', author='Arjun Arkal Rao', @@ -117,10 +116,10 @@ def run_tests(self): license='Apache', install_requires=[ 'PyYAML', - 'pandas==0.19.2' + 'pandas' ], tests_require=[ - 'pytest==2.8.3'], + 'pytest'], test_suite='protect', entry_points={ 'console_scripts': [ diff --git a/src/__init__.py b/src/__init__.py index ba9b9130..dabfb0af 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,4 +13,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + diff --git a/src/protect/ProTECT_config.yaml b/src/protect/ProTECT_config.yaml new file mode 100644 index 00000000..239a42d0 --- /dev/null +++ b/src/protect/ProTECT_config.yaml @@ -0,0 +1,179 @@ +## Copyright 2016 UCSC Computational Genomics Lab +## Original contributor: Arjun Arkal Rao +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +#################################################################################################### +#################################################################################################### +## This is the input parameter file for the precision immuno pipeline. The parameters for each of +## the tools is provided here. The file is written in the YAML format. A nice description of the +## format can be found at http://docs.ansible.com/ansible/YAMLSyntax.html +## +## You can add comments anywhere in this file by prefixing it with a '#' +## +## Unless otherwise mentioned, all fields must be filled. +## +#################################################################################################### +#################################################################################################### + +# Any number of patients/samples can be listed here +patients: + # Each group starts with the patient ID + PRTCT-01: + # The paths should point to the forward read of the pair. The pipeline assumes that the + # forward and reverse read files have the same prefix + tumor_dna_fastq_1: /home/dranion/treehouse/gms/hcc1395_1tenth_percent/sort/gerald_D1VCPACXX_1_R1.fastq + tumor_dna_fastq_2: /home/dranion/treehouse/gms/hcc1395_1tenth_percent/sort/gerald_D1VCPACXX_1_R2.fastq + normal_dna_fastq_1: /home/dranion/treehouse/gms/hcc1395_1tenth_percent/sort/gerald_D1VCPACXX_6_R1.fastq + normal_dna_fastq_2: /home/dranion/treehouse/gms/hcc1395_1tenth_percent/sort/gerald_D1VCPACXX_6_R2.fastq + tumor_rna_fastq_1: /home/dranion/treehouse/gms/hcc1395_1tenth_percent/sort/gerald_C1TD1ACXX_8_ACAGTG_R1.fastq + tumor_rna_fastq_2: /home/dranion/treehouse/gms/hcc1395_1tenth_percent/sort/gerald_C1TD1ACXX_8_ACAGTG_R2.fastq + tumor_type: SKCM + # ssec_encrypted: False + # filter_for_OxoG: False + # PRTCT-02: + # The paths can also be to directories on S3 as + #tumor_dna_fastq_1: S3://bucket/path/to/1.fastq.gz + #normal_dna_fastq_1: S3://bucket/path/to/1.fastq.gz + #tumor_rna_fastq_1: https://S3-.awsamazon.com/bucket/path/to/1.fastq.gz + +# These are options that are used by most tools +Universal_Options: + dockerhub: aarjunrao + java_Xmx: 20G + reference_build: hg38 # Acceptable options are hg38, hg38, GRCh37, GRCh38 + # sse_key: /path/to/master.key # Path to the AWS master key. Required if using AWS else optional + # sse_key_is_master: True # True or False. Required if using AWS else optional + # gdc_download_token: /path/to/token.txt # Path to the user's GDC download token. + storage_location: Local # Local or aws: for where the output must go + #storage_location: aws:protect-run-xyz + output_folder: /home/dranion/Flashdrive/project-results # Path to where the output must go. + #mail_to: test.email@host.com # Email for sending success report. + + +# These options are for each module. You probably don't need to change any of this! +alignment: + cutadapt: + a: AGATCGGAAGAG + A: AGATCGGAAGAG + # version: 1.9.1 + star: + type: star # use starlong if your reads are > 150bp + index: /home/dranion/Flashdrive/protect-data/star_with_fusion_100bp_readlen_indexes.tar.gz # Use star_without if you set star_fusion = False + # version: 2.5.2b + bwa: + index: /home/dranion/Flashdrive/protect-data/bwa_index.tar.gz + # version: 0.7.9a + post: + samtools: + # version: 1.2 + picard: + # version: 1.135 + +expression_estimation: + rsem: + index: /home/dranion/Flashdrive/protect-data/rsem_index.tar.gz + # version: 1.2.0 + +mutation_calling: + indexes: + chromosomes: canonical_chr, chrM + genome_fasta: /home/dranion/Flashdrive/protect-data/hg38.fa.tar.gz + genome_fai: /home/dranion/Flashdrive/protect-data/hg38.fa.fai.tar.gz + genome_dict: /home/dranion/Flashdrive/protect-data/hg38.dict.tar.gz + cosmic_vcf: /home/dranion/Flashdrive/protect-data/CosmicCodingMuts.vcf.tar.gz + cosmic_idx: /home/dranion/Flashdrive/protect-data/CosmicCodingMuts.vcf.idx.tar.gz + dbsnp_vcf: /home/dranion/Flashdrive/protect-data/dbsnp_coding.vcf.gz + dbsnp_idx: /home/dranion/Flashdrive/protect-data/dbsnp_coding.vcf.idx.tar.gz + dbsnp_tbi: /home/dranion/Flashdrive/protect-data/dbsnp_coding.vcf.gz.tbi + mutect: + java_Xmx: 2G + # version: 1.1.7 + muse: + # version: 1.0rc_submission_b391201 + radia: + cosmic_beds: /home/dranion/Flashdrive/protect-data/radia_cosmic.tar.gz + dbsnp_beds: /home/dranion/Flashdrive/protect-data/radia_dbsnp.tar.gz + retrogene_beds: /home/dranion/Flashdrive/protect-data/radia_retrogenes.tar.gz + pseudogene_beds: /home/dranion/Flashdrive/protect-data/radia_pseudogenes.tar.gz + gencode_beds: /home/dranion/Flashdrive/protect-data/radia_gencode.tar.gz + # version: 398366ef07b5911d8082ed61cbf03d487a41f286 + somaticsniper: + # version: 1.0.4 + samtools: + # version: 0.1.8 + bam_readcount: + # version: 0.7.4 + star_fusion: + #run: True + #version: 1.0.0 + fusion_inspector: + #run_trinity: True + #version: 1.0.1 + strelka: + # version: 1.0.15 + config_file: /home/dranion/Flashdrive/protect-data/strelka_bwa_WXS_config.ini.tar.gz + + +mutation_annotation: + snpeff: + index: /home/dranion/Flashdrive/protect-data/snpeff_index.tar.gz + # version: 3.6 + java_Xmx: 20G + +mutation_translation: + transgene: + gencode_peptide_fasta : /home/dranion/Flashdrive/protect-data/gencode.v25.pc_translations_NOPARY.fa.tar.gz + gencode_transcript_fasta : /home/dranion/Flashdrive/protect-data/gencode.v25.pc_transcripts_NOPARY.fa.tar.gz + gencode_annotation_gtf : /home/dranion/Flashdrive/protect-data/gencode.v25.annotation_NOPARY.gtf.tar.gz + genome_fasta : /home/dranion/Flashdrive/protect-data/hg38.fa.tar.gz + # version: 2.2.2 + +haplotyping: + phlat: + index: /home/dranion/Flashdrive/protect-data/phlat_index.tar.gz + # version: 1.0 + +mhc_peptide_binding: + mhci: + method_file: /home/dranion/Flashdrive/protect-data/mhci_restrictions.json.tar.gz + pred: IEDB_recommended + # version: 2.13 + mhcii: + method_file: /home/dranion/Flashdrive/protect-data/mhcii_restrictions.json.tar.gz + pred: IEDB_recommended + # version: 2.13 + netmhciipan: + # version: 3.1 + +prediction_ranking: + rankboost: + mhci_args: + npa: 0.0 + nph: 0.0 + nMHC: 0.32 + TPM: 0.0 + overlap: 0.68 + tndelta: 0.0 + mhcii_args: + npa: 0.2 + nph: 0.2 + nMHC: 0.2 + TPM: 0.2 + tndelta: 0.2 + # version: 2.0.3 + +reports: + mhc_pathways_file: /home/dranion/Flashdrive/protect-data/mhc_pathways.tsv.tar.gz + itx_resistance_file: /home/dranion/Flashdrive/protect-data/itx_resistance.tsv.tar.gz + immune_resistance_pathways_file: /home/dranion/Flashdrive/protect-data/immune_resistance_pathways.json.tar.gz + car_t_targets_file: /home/dranion/Flashdrive/protect-data/car_t_targets.tsv.tar.gz diff --git a/src/protect/__init__.py b/src/protect/__init__.py index ba9b9130..dabfb0af 100644 --- a/src/protect/__init__.py +++ b/src/protect/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,4 +13,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + diff --git a/src/protect/addons/__init__.py b/src/protect/addons/__init__.py index ba9b9130..dabfb0af 100644 --- a/src/protect/addons/__init__.py +++ b/src/protect/addons/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,4 +13,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + diff --git a/src/protect/addons/assess_car_t_validity.py b/src/protect/addons/assess_car_t_validity.py index f28a2de4..e49a4469 100644 --- a/src/protect/addons/assess_car_t_validity.py +++ b/src/protect/addons/assess_car_t_validity.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function + from protect.addons.common import TCGAToGTEx from protect.common import export_results, get_files_from_filestore, untargz from protect.haplotyping.phlat import parse_phlat_file diff --git a/src/protect/addons/assess_immunotherapy_resistance.py b/src/protect/addons/assess_immunotherapy_resistance.py index 568aacf9..6f96530b 100644 --- a/src/protect/addons/assess_immunotherapy_resistance.py +++ b/src/protect/addons/assess_immunotherapy_resistance.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function + from protect.addons.common import TCGAToGTEx from protect.common import export_results, get_files_from_filestore, untargz diff --git a/src/protect/addons/assess_mhc_pathway.py b/src/protect/addons/assess_mhc_pathway.py index 149df4ba..d5ee00d2 100644 --- a/src/protect/addons/assess_mhc_pathway.py +++ b/src/protect/addons/assess_mhc_pathway.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function + from collections import Counter from protect.addons.common import TCGAToGTEx from protect.common import export_results, get_files_from_filestore, untargz @@ -71,6 +71,7 @@ def assess_mhc_genes(job, gene_expression, rna_haplotype, univ_options, reports_ # Read the background file background_df = pd.read_table(input_files['mhc_pathways.tsv'], index_col=0, header=0) + print(background_df) # Parse the rna phlat file if rna_haplotype is not None: @@ -81,7 +82,7 @@ def assess_mhc_genes(job, gene_expression, rna_haplotype, univ_options, reports_ # Read the patient gene values into a dictionary gene_expressions = pd.read_table(input_files['rsem_quant.tsv'], index_col=0, header=0) - gene_expressions = Counter({x.split('.')[0]: y for x, y in gene_expressions['TPM'].to_dict().items()}) + gene_expressions = Counter({x.split('.')[0]: y for x, y in list(gene_expressions['TPM'].to_dict().items())}) # Print the report roles = {x for x in background_df['Roles'].values if ',' not in x} with open('mhc_pathway_report.txt', 'w') as mpr: @@ -119,7 +120,10 @@ def assess_mhc_genes(job, gene_expression, rna_haplotype, univ_options, reports_ result, 2, result), file=mpr) for ensg in role_df.index: - ensgName = background_df.ix[ensg, 'Name'] + #ix was depricated in 0.20.0 + #ensgName = background_df.ix[ensg, 'Name'] + # i haven't worked with pandas before, but this *should* be the same? + ensgName = background_df.loc[ensg, 'Name'] b_vals = {} for bkg in b_types: val = "{0:.2f}".format(role_df.loc[ensg].get(b_types[bkg], default='NA')) diff --git a/src/protect/addons/common.py b/src/protect/addons/common.py index 8566ae83..5dc8be99 100644 --- a/src/protect/addons/common.py +++ b/src/protect/addons/common.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function + from collections import Counter diff --git a/src/protect/alignment/__init__.py b/src/protect/alignment/__init__.py index ba9b9130..dabfb0af 100644 --- a/src/protect/alignment/__init__.py +++ b/src/protect/alignment/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,4 +13,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + diff --git a/src/protect/alignment/common.py b/src/protect/alignment/common.py index b959fc37..70cb72a0 100644 --- a/src/protect/alignment/common.py +++ b/src/protect/alignment/common.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + from math import ceil from protect.common import docker_call, docker_path, export_results, get_files_from_filestore diff --git a/src/protect/alignment/dna.py b/src/protect/alignment/dna.py index 105fa3d8..767065a1 100644 --- a/src/protect/alignment/dna.py +++ b/src/protect/alignment/dna.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import, print_function + from math import ceil from protect.alignment.common import index_bamfile, index_disk @@ -128,7 +128,7 @@ def run_bwa(job, fastqs, sample_type, univ_options, bwa_options): input_files[read_file + gz] = input_files[read_file] + gz # Untar the index input_files['bwa_index'] = untargz(input_files['bwa_index.tar.gz'], work_dir) - input_files = {key: docker_path(path) for key, path in input_files.items()} + input_files = {key: docker_path(path) for key, path in list(input_files.items())} parameters = ['mem', '-t', str(bwa_options['n']), diff --git a/src/protect/alignment/rna.py b/src/protect/alignment/rna.py index fe8fe86b..aeb38320 100644 --- a/src/protect/alignment/rna.py +++ b/src/protect/alignment/rna.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import, print_function + from collections import defaultdict from math import ceil @@ -94,7 +94,7 @@ def run_star(job, fastqs, univ_options, star_options): if os.path.exists(star_fusion_idx): input_files['star_index'] = star_fusion_idx - input_files = {key: docker_path(path, work_dir=work_dir) for key, path in input_files.items()} + input_files = {key: docker_path(path, work_dir=work_dir) for key, path in list(input_files.items())} # Using recommended STAR-Fusion parameters: # https://github.com/STAR-Fusion/STAR-Fusion/wiki diff --git a/src/protect/binding_prediction/__init__.py b/src/protect/binding_prediction/__init__.py index ba9b9130..dabfb0af 100644 --- a/src/protect/binding_prediction/__init__.py +++ b/src/protect/binding_prediction/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,4 +13,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + diff --git a/src/protect/binding_prediction/common.py b/src/protect/binding_prediction/common.py index 3bead87e..2c58a9e9 100644 --- a/src/protect/binding_prediction/common.py +++ b/src/protect/binding_prediction/common.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import, print_function + from collections import defaultdict from protect.binding_prediction.mhci import predict_mhci_binding @@ -108,7 +108,7 @@ def spawn_antigen_predictors(job, transgened_files, phlat_files, univ_options, m mhci_preds[(allele, peplen)] = mhci_job.addChildJobFn( predict_normal_binding, mhci_job.rv(), - {x: y for x, y in pept_files.items() if peplen in x}, + {x: y for x, y in list(pept_files.items()) if peplen in x}, allele, peplen, univ_options, @@ -125,7 +125,7 @@ def spawn_antigen_predictors(job, transgened_files, phlat_files, univ_options, m mhcii_preds[(allele, 15)] = mhcii_job.addFollowOnJobFn( predict_normal_binding, mhcii_job.rv(), - {x: y for x, y in pept_files.items() if '15' in x}, + {x: y for x, y in list(pept_files.items()) if '15' in x}, allele, '15', univ_options, @@ -145,8 +145,8 @@ def read_fastas(input_files): :return: The read fastas in a dictionary of tuples :rtype: dict """ - tumor_file = [y for x, y in input_files.items() if x.startswith('T')][0] - normal_file = [y for x, y in input_files.items() if x.startswith('N')][0] + tumor_file = [y for x, y in list(input_files.items()) if x.startswith('T')][0] + normal_file = [y for x, y in list(input_files.items()) if x.startswith('N')][0] output_files = defaultdict(list) output_files = _read_fasta(tumor_file, output_files) num_entries = len(output_files) @@ -343,7 +343,7 @@ def _get_normal_peptides(job, mhc_df, iars, peplen): peplen = int(peplen) normal_peptides = [] for pred in mhc_df.itertuples(): - containing_iars = [i for i, sl in iars.items() if pred.pept in sl[0]] + containing_iars = [i for i, sl in list(iars.items()) if pred.pept in sl[0]] assert len(containing_iars) != 0, "No IARS contained the peptide" if len(iars[containing_iars[0]]) == 1: # This is a fusion and has no corresponding normal @@ -351,7 +351,7 @@ def _get_normal_peptides(job, mhc_df, iars, peplen): else: # If there are multiple IARs, they all or none of them have to have a corresponding # normal. - if len(set([len(y) for x, y in iars.items() if x in containing_iars])) != 1: + if len(set([len(y) for x, y in list(iars.items()) if x in containing_iars])) != 1: job.fileStore.logToMaster('Some IARS were found to contain the substring but were' 'inconsistent with the presence of a corresponding ' 'normal.') @@ -594,7 +594,7 @@ def print_mhc_peptide(neoepitope_info, peptides, pepmap, outfile, netmhc=False): if netmhc: peptide_names = [neoepitope_info.peptide_name] else: - peptide_names = [x for x, y in peptides.items() if neoepitope_info.pept in y] + peptide_names = [x for x, y in list(peptides.items()) if neoepitope_info.pept in y] # Convert named tuple to dict so it can be modified neoepitope_info = neoepitope_info._asdict() # Handle fusion peptides (They are characterized by having all N's as the normal partner) diff --git a/src/protect/binding_prediction/mhci.py b/src/protect/binding_prediction/mhci.py index 2708771d..f5679fbd 100644 --- a/src/protect/binding_prediction/mhci.py +++ b/src/protect/binding_prediction/mhci.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import, print_function + from protect.common import docker_call, get_files_from_filestore, read_peptide_file diff --git a/src/protect/binding_prediction/mhcii.py b/src/protect/binding_prediction/mhcii.py index 3e6f93a0..91db1f62 100644 --- a/src/protect/binding_prediction/mhcii.py +++ b/src/protect/binding_prediction/mhcii.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import, print_function + from protect.common import docker_call, get_files_from_filestore, read_peptide_file diff --git a/src/protect/common.py b/src/protect/common.py old mode 100644 new mode 100755 index 17d72b5e..2579f909 --- a/src/protect/common.py +++ b/src/protect/common.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -21,16 +21,17 @@ Program info can be found in the docstring of the main function. Details can also be obtained by running the script with -h . """ -from __future__ import print_function + from collections import defaultdict from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from urlparse import urlparse - +from urllib.parse import urlparse +from io import IOBase import errno import gzip import logging +import shutil import os import re import smtplib @@ -38,7 +39,7 @@ import subprocess import sys import tarfile -import urllib2 +import urllib.request, urllib.error, urllib.parse import uuid @@ -53,7 +54,7 @@ def get_files_from_filestore(job, files, work_dir, docker=False): :return: Dict of files: (optionallly docker-friendly) fileepaths :rtype: dict """ - for name in files.keys(): + for name in list(files.keys()): outfile = job.fileStore.readGlobalFile(files[name], '/'.join([work_dir, name])) # If the files will be sent to docker, we will mount work_dir to the container as /data and # we want the /data prefixed path to the file @@ -98,7 +99,7 @@ def docker_call(tool, tool_parameters, work_dir, java_xmx=None, outfile=None, # If an outifle has been provided, then ensure that it is of type file, it is writeable, and # that it is open. if outfile: - assert isinstance(outfile, file), 'outfile was not passsed a file' + assert isinstance(outfile, IOBase), 'outfile was not passsed a file' assert outfile.mode in ['w', 'a', 'wb', 'ab'], 'outfile not writeable' assert not outfile.closed, 'outfile is closed' # If the call is interactive, set intereactive to -i @@ -110,7 +111,7 @@ def docker_call(tool, tool_parameters, work_dir, java_xmx=None, outfile=None, docker_tool = ''.join([dockerhub, '/', tool, ':', tool_version]) # Get the docker image on the worker if needed call = ['docker', 'images'] - dimg_rv = subprocess.check_output(call) + dimg_rv = subprocess.check_output(call).decode('utf-8') existing_images = [':'.join(x.split()[0:2]) for x in dimg_rv.splitlines() if x.startswith(dockerhub)] @@ -160,7 +161,7 @@ def untargz(input_targz_file, untar_to_dir): return return_value -def gunzip(input_gzip_file, block_size=1024): +def gunzip(input_gzip_file, block_size=2048): """ Gunzips the input file to the same directory @@ -168,16 +169,22 @@ def gunzip(input_gzip_file, block_size=1024): :return: path to the gunzipped file :rtype: str """ + #TODO THIS IS WHERE THE PROBLEM IS? WHAT IS THIS EVEN DOING + print("DEBUG: gunzip funciton") assert os.path.splitext(input_gzip_file)[1] == '.gz' assert is_gzipfile(input_gzip_file) with gzip.open(input_gzip_file) as infile: - with open(os.path.splitext(input_gzip_file)[0], 'w') as outfile: - while True: - block = infile.read(block_size) - if block == '': - break - else: - outfile.write(block) + print("DEBUG: OPENING GZ FILE") + with open(os.path.splitext(input_gzip_file)[0], 'wb') as outfile: + shutil.copyfileobj(infile, outfile) + #why copy a file like this?? tf + #while True: + # block = str(infile.read(block_size)) + # if block == '': + # break + # else: + # print("DEBUG: writing outfile block" + str(block)) + # outfile.write(block) return outfile.name @@ -197,7 +204,7 @@ def is_gzipfile(filename): 'point to a file.' with open(filename, 'rb') as in_f: start_of_file = in_f.read(3) - if start_of_file == '\x1f\x8b\x08': + if start_of_file == b'\x1f\x8b\x08': return True else: return False @@ -355,8 +362,8 @@ def get_file_from_url(job, any_url, encryption_key=None, per_file_encryption=Tru url = any_url parsed_url = urlparse(any_url) try: - response = urllib2.urlopen(url) - except urllib2.HTTPError: + response = urllib.request.urlopen(url) + except urllib.error.HTTPError: if parsed_url.netloc.startswith(('s3', 'S3')): job.fileStore.logToMaster("Detected https link is for an encrypted s3 file.") return get_file_from_s3(job, any_url, encryption_key=encryption_key, @@ -431,7 +438,7 @@ def export_results(job, fsid, file_name, univ_options, subfolder=None): # Handle Local try: # Create the directory if required - os.makedirs(output_folder, 0755) + os.makedirs(output_folder, 0o755) except OSError as err: if err.errno != errno.EEXIST: raise @@ -455,7 +462,7 @@ def delete_fastqs(job, patient_dict): :param dict patient_dict: Dict of list of input fastqs """ - for key in patient_dict.keys(): + for key in list(patient_dict.keys()): if 'fastq' not in key: continue job.fileStore.logToMaster('Deleting "%s:%s" ' % (patient_dict['patient_id'], key) + @@ -472,10 +479,10 @@ def delete_bams(job, bams, patient_id): :param dict bams: Dict of bam and bai files :param str patient_id: The ID of the patient for logging purposes. """ - bams = {b: v for b, v in bams.items() + bams = {b: v for b, v in list(bams.items()) if (b.endswith('.bam') or b.endswith('.bai')) and v is not None} if bams: - for key, val in bams.items(): + for key, val in list(bams.items()): job.fileStore.logToMaster('Deleting "%s" for patient "%s".' % (key, patient_id)) job.fileStore.deleteGlobalFile(val) elif 'rna_genome' in bams: @@ -600,7 +607,11 @@ def canonical_chrom_sorted(in_chroms): if 'MT' in in_chroms: in_chroms[in_chroms.index('MT')] = 'M' mt = True - in_chroms = sorted(in_chroms, key=lambda c: int(c) if c not in ('X', 'Y', 'M') else c) + num_in_chroms = sorted(filter(str.isnumeric, in_chroms), + key = lambda c: int(c)) + chr_in_chroms = sorted(filter(str.isalpha, in_chroms)) + in_chroms = num_in_chroms.copy() + in_chroms.extend(chr_in_chroms) try: m_index = in_chroms.index('M') except ValueError: @@ -656,4 +667,4 @@ def dummy_job(job, return_value): :param return_value: Any object of any pickle-able type :return: return_value """ - return return_value \ No newline at end of file + return return_value diff --git a/src/protect/expression_profiling/__init__.py b/src/protect/expression_profiling/__init__.py index ba9b9130..dabfb0af 100644 --- a/src/protect/expression_profiling/__init__.py +++ b/src/protect/expression_profiling/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,4 +13,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + diff --git a/src/protect/expression_profiling/rsem.py b/src/protect/expression_profiling/rsem.py index cc44699b..91c329d1 100644 --- a/src/protect/expression_profiling/rsem.py +++ b/src/protect/expression_profiling/rsem.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function + from math import ceil from protect.common import (docker_call, @@ -75,10 +75,10 @@ def run_rsem(job, rna_bam, univ_options, rsem_options): input_files = get_files_from_filestore(job, input_files, work_dir, docker=False) input_files['rsem_index'] = untargz(input_files['rsem_index.tar.gz'], work_dir) - input_files = {key: docker_path(path) for key, path in input_files.items()} + input_files = {key: docker_path(path) for key, path in list(input_files.items())} parameters = ['--paired-end', - '-p', str(rsem_options['n']), + '-p', str(20), '--bam', input_files['star_transcriptome.bam'], '--no-bam-output', diff --git a/src/protect/haplotyping/__init__.py b/src/protect/haplotyping/__init__.py index ba9b9130..dabfb0af 100644 --- a/src/protect/haplotyping/__init__.py +++ b/src/protect/haplotyping/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,4 +13,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + diff --git a/src/protect/haplotyping/phlat.py b/src/protect/haplotyping/phlat.py index 1d34c9c1..43e7bdf8 100644 --- a/src/protect/haplotyping/phlat.py +++ b/src/protect/haplotyping/phlat.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import, print_function + from collections import defaultdict from math import ceil @@ -51,6 +51,7 @@ def run_phlat(job, fastqs, sample_type, univ_options, phlat_options): 'input_2.fastq': fastqs[1], 'phlat_index.tar.gz': phlat_options['index']} input_files = get_files_from_filestore(job, input_files, work_dir, docker=False) + print(input_files) # Handle gzipped files gz = '.gz' if is_gzipfile(input_files['input_1.fastq']) else '' if gz: @@ -59,7 +60,7 @@ def run_phlat(job, fastqs, sample_type, univ_options, phlat_options): input_files[read_file + gz] = input_files[read_file] + gz # Untar the index input_files['phlat_index'] = untargz(input_files['phlat_index.tar.gz'], work_dir) - input_files = {key: docker_path(path) for key, path in input_files.items()} + input_files = {key: docker_path(path) for key, path in list(input_files.items())} parameters = ['-1', input_files['input_1.fastq' + gz], '-2', input_files['input_2.fastq' + gz], @@ -68,7 +69,7 @@ def run_phlat(job, fastqs, sample_type, univ_options, phlat_options): '-tag', sample_type, '-e', '/home/phlat-1.0', # Phlat directory home '-o', '/data', # Output directory - '-p', str(phlat_options['n'])] # Number of threads + '-p', str(10)] # Number of threads docker_call(tool='phlat', tool_parameters=parameters, work_dir=work_dir, dockerhub=univ_options['dockerhub'], tool_version=phlat_options['version']) output_file = job.fileStore.writeGlobalFile(''.join([work_dir, '/', sample_type, '_HLA.sum'])) @@ -182,13 +183,13 @@ def most_probable_alleles(allele_list): except KeyError: all_alleles[allele] = [float(pvalue)] # If there are less than 2 alleles, report all - if len(all_alleles.keys()) <= 2: - return all_alleles.keys() + if len(list(all_alleles.keys())) <= 2: + return list(all_alleles.keys()) # Else, get the two with most evidence. Evidence is gauged by # a) How many files (of the 3) thought that Allele was present # b) In a tie, who has a lower avg p value # In the lambda function, if 2 alleles have the same number of calls, the sum of the p values is # a measure of the avg because avg = sum / n and n is equal in both of them. else: - return sorted(all_alleles.keys(), + return sorted(list(all_alleles.keys()), key=lambda x: (-len(all_alleles[x]), sum(all_alleles[x])))[0:2] diff --git a/src/protect/mutation_annotation/__init__.py b/src/protect/mutation_annotation/__init__.py index ba9b9130..dabfb0af 100644 --- a/src/protect/mutation_annotation/__init__.py +++ b/src/protect/mutation_annotation/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,4 +13,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + diff --git a/src/protect/mutation_annotation/snpeff.py b/src/protect/mutation_annotation/snpeff.py index 43ed03c0..ab54dc24 100644 --- a/src/protect/mutation_annotation/snpeff.py +++ b/src/protect/mutation_annotation/snpeff.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import, print_function + from math import ceil from protect.common import (docker_call, @@ -45,7 +45,7 @@ def run_snpeff(job, merged_mutation_file, univ_options, snpeff_options): 'snpeff_index.tar.gz': snpeff_options['index']} input_files = get_files_from_filestore(job, input_files, work_dir, docker=False) input_files['snpeff_index'] = untargz(input_files['snpeff_index.tar.gz'], work_dir) - input_files = {key: docker_path(path) for key, path in input_files.items()} + input_files = {key: docker_path(path) for key, path in list(input_files.items())} parameters = ['eff', '-dataDir', input_files['snpeff_index'], diff --git a/src/protect/mutation_calling/__init__.py b/src/protect/mutation_calling/__init__.py index ba9b9130..dabfb0af 100644 --- a/src/protect/mutation_calling/__init__.py +++ b/src/protect/mutation_calling/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,4 +13,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + diff --git a/src/protect/mutation_calling/common.py b/src/protect/mutation_calling/common.py index 947426d2..cac561d4 100644 --- a/src/protect/mutation_calling/common.py +++ b/src/protect/mutation_calling/common.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import, print_function + from collections import defaultdict from protect.common import chrom_sorted, export_results, get_files_from_filestore, untargz @@ -63,7 +63,7 @@ def run_mutation_aggregator(job, mutation_results, univ_options): """ # Setup an input data structure for the merge function out = {} - for chrom in mutation_results['mutect'].keys(): + for chrom in list(mutation_results['mutect'].keys()): out[chrom] = job.addChildJobFn(merge_perchrom_mutations, chrom, mutation_results, univ_options).rv() merged_snvs = job.addFollowOnJobFn(merge_perchrom_vcfs, out, 'merged', univ_options) @@ -110,7 +110,7 @@ def merge_perchrom_mutations(job, chrom, mutations, univ_options): accepted_hits = defaultdict(dict) - for mut_type in vcf_processor.keys(): + for mut_type in list(vcf_processor.keys()): # Get input files perchrom_mutations = {caller: vcf_processor[mut_type][caller](job, mutations[caller][chrom], work_dir, univ_options) @@ -119,12 +119,12 @@ def merge_perchrom_mutations(job, chrom, mutations, univ_options): perchrom_mutations['strelka'] = perchrom_mutations['strelka_' + mut_type] perchrom_mutations.pop('strelka_' + mut_type) # Read in each file to a dict - vcf_lists = {caller: read_vcf(vcf_file) for caller, vcf_file in perchrom_mutations.items()} - all_positions = list(set(itertools.chain(*vcf_lists.values()))) + vcf_lists = {caller: read_vcf(vcf_file) for caller, vcf_file in list(perchrom_mutations.items())} + all_positions = list(set(itertools.chain(*list(vcf_lists.values())))) for position in sorted(all_positions): - hits = {caller: position in vcf_lists[caller] for caller in perchrom_mutations.keys()} + hits = {caller: position in vcf_lists[caller] for caller in list(perchrom_mutations.keys())} if sum(hits.values()) >= majority[mut_type]: - callers = ','.join([caller for caller, hit in hits.items() if hit]) + callers = ','.join([caller for caller, hit in list(hits.items()) if hit]) assert position[1] not in accepted_hits[position[0]] accepted_hits[position[0]][position[1]] = (position[2], position[3], callers) @@ -133,7 +133,7 @@ def merge_perchrom_mutations(job, chrom, mutations, univ_options): print('##INFO=') @@ -297,7 +297,8 @@ def split_fusion_transcript(annotation_path, transcripts): forward = 'ACGTN' reverse = 'TGCAN' - trans = string.maketrans(forward, reverse) + # string.maketrans depricated, so each type has its own maketrans methods + trans = str.maketrans(forward, reverse) # Pull in assembled transcript annotation five_pr_splits = collections.defaultdict(dict) @@ -435,13 +436,13 @@ def reformat_star_fusion_output(job, score = 'Junction:%s-Spanning:%s' % (record.JunctionReadCount, record.SpanningFragCount) # Add empty sequences in case Trinity doesn't output one - if len(five_pr_splits[fusion].keys()) == 0: + if len(list(five_pr_splits[fusion].keys())) == 0: five_pr_splits[fusion]['N/A'] = '.' - if len(three_pr_splits[fusion].keys()) == 0: + if len(list(three_pr_splits[fusion].keys())) == 0: three_pr_splits[fusion]['N/A'] = '.' - for transcript_id in five_pr_splits[fusion].keys(): + for transcript_id in list(five_pr_splits[fusion].keys()): five_prime_seq = five_pr_splits[fusion][transcript_id] three_prime_seq = three_pr_splits[fusion][transcript_id] diff --git a/src/protect/mutation_calling/indel.py b/src/protect/mutation_calling/indel.py index 7c8990da..fc05981a 100644 --- a/src/protect/mutation_calling/indel.py +++ b/src/protect/mutation_calling/indel.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function + def run_indel_caller(job, tumor_bam, normal_bam, univ_options, indel_options): diff --git a/src/protect/mutation_calling/muse.py b/src/protect/mutation_calling/muse.py index 28be1c6b..fc837067 100644 --- a/src/protect/mutation_calling/muse.py +++ b/src/protect/mutation_calling/muse.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function + from collections import defaultdict from math import ceil @@ -125,7 +125,7 @@ def run_muse_perchrom(job, tumor_bam, normal_bam, univ_options, muse_options, ch for key in ('genome.fa', 'genome.fa.fai'): input_files[key] = untargz(input_files[key + '.tar.gz'], work_dir) - input_files = {key: docker_path(path) for key, path in input_files.items()} + input_files = {key: docker_path(path) for key, path in list(input_files.items())} output_prefix = os.path.join(work_dir, chrom) @@ -162,9 +162,9 @@ def run_muse_sump_perchrom(job, muse_output, univ_options, muse_options, chrom): tbi = os.path.splitext(input_files['dbsnp_coding.vcf.gz.tbi.tmp'])[0] time.sleep(2) shutil.copy(input_files['dbsnp_coding.vcf.gz.tbi.tmp'], tbi) - os.chmod(tbi, 0777) + os.chmod(tbi, 0o777) open(tbi, 'a').close() - input_files = {key: docker_path(path) for key, path in input_files.items()} + input_files = {key: docker_path(path) for key, path in list(input_files.items())} output_file = ''.join([work_dir, '/', chrom, '.vcf']) parameters = ['sump', diff --git a/src/protect/mutation_calling/mutect.py b/src/protect/mutation_calling/mutect.py index eb1a556f..edcbdd95 100644 --- a/src/protect/mutation_calling/mutect.py +++ b/src/protect/mutation_calling/mutect.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function + from collections import defaultdict from math import ceil @@ -76,6 +76,8 @@ def run_mutect(job, tumor_bam, normal_bam, univ_options, mutect_options): :rtype: dict """ # Get a list of chromosomes to handle + job.fileStore.logToMaster('DEBUG: running mutect, log to master') + print("DEBUG: running mutect") if mutect_options['chromosomes']: chromosomes = mutect_options['chromosomes'] else: @@ -105,7 +107,9 @@ def run_mutect_perchrom(job, tumor_bam, normal_bam, univ_options, mutect_options :return: fsID for the chromsome vcf :rtype: toil.fileStore.FileID """ + work_dir = os.getcwd() + input_files = { 'tumor.bam': tumor_bam['tumor_dna_fix_pg_sorted.bam'], 'tumor.bam.bai': tumor_bam['tumor_dna_fix_pg_sorted.bam.bai'], @@ -119,13 +123,17 @@ def run_mutect_perchrom(job, tumor_bam, normal_bam, univ_options, mutect_options 'dbsnp.vcf.gz': mutect_options['dbsnp_vcf'], 'dbsnp.vcf.idx.tar.gz': mutect_options['dbsnp_idx']} input_files = get_files_from_filestore(job, input_files, work_dir, docker=False) + # dbsnp.vcf should be bgzipped, but all others should be tar.gz'd + job.fileStore.logToMaster('unzipping dbsnp.vcf') input_files['dbsnp.vcf'] = gunzip(input_files['dbsnp.vcf.gz']) + #input_files['dbsnp.vcf'] = + job.fileStore.logToMaster('unzipped dbsnp') for key in ('genome.fa', 'genome.fa.fai', 'genome.dict', 'cosmic.vcf', 'cosmic.vcf.idx', 'dbsnp.vcf.idx'): input_files[key] = untargz(input_files[key + '.tar.gz'], work_dir) - input_files = {key: docker_path(path) for key, path in input_files.items()} - + input_files = {key: docker_path(path) for key, path in list(input_files.items())} + job.fileStore.logToMaster('configured input files for docker') mutout = ''.join([work_dir, '/', chrom, '.out']) mutvcf = ''.join([work_dir, '/', chrom, '.vcf']) parameters = ['-R', input_files['genome.fa'], @@ -141,9 +149,11 @@ def run_mutect_perchrom(job, tumor_bam, normal_bam, univ_options, mutect_options ] java_xmx = mutect_options['java_Xmx'] if mutect_options['java_Xmx'] \ else univ_options['java_Xmx'] + job.fileStore.logToMaster('starting mutect docker call') docker_call(tool='mutect', tool_parameters=parameters, work_dir=work_dir, dockerhub=univ_options['dockerhub'], java_xmx=java_xmx, tool_version=mutect_options['version']) + job.fileStore.logToMaster('finshed docker, outputting now') output_file = job.fileStore.writeGlobalFile(mutvcf) export_results(job, output_file, mutvcf, univ_options, subfolder='mutations/mutect') job.fileStore.logToMaster('Ran MuTect on %s:%s successfully' % (univ_options['patient'], chrom)) @@ -161,6 +171,7 @@ def process_mutect_vcf(job, mutect_vcf, work_dir, univ_options): :rtype: str """ mutect_vcf = job.fileStore.readGlobalFile(mutect_vcf) + job.fileStore.logToMaster('process mutect vcf start') with open(mutect_vcf, 'r') as infile, open(mutect_vcf + 'mutect_parsed.tmp', 'w') as outfile: for line in infile: diff --git a/src/protect/mutation_calling/radia.py b/src/protect/mutation_calling/radia.py index 178bd8dd..9ba7b863 100644 --- a/src/protect/mutation_calling/radia.py +++ b/src/protect/mutation_calling/radia.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function + from collections import defaultdict from math import ceil @@ -87,7 +87,7 @@ def run_radia(job, rna_bam, tumor_bam, normal_bam, univ_options, radia_options): +- 'chrM': fsID :rtype: dict """ - if 'rna_genome' in rna_bam.keys(): + if 'rna_genome' in list(rna_bam.keys()): rna_bam = rna_bam['rna_genome'] elif set(rna_bam.keys()) == {'rna_genome_sorted.bam', 'rna_genome_sorted.bam.bai'}: pass @@ -151,7 +151,7 @@ def run_radia_perchrom(job, bams, univ_options, radia_options, chrom): for key in ('genome.fa', 'genome.fa.fai'): input_files[key] = untargz(input_files[key + '.tar.gz'], work_dir) - input_files = {key: docker_path(path) for key, path in input_files.items()} + input_files = {key: docker_path(path) for key, path in list(input_files.items())} radia_output = ''.join([work_dir, '/radia_', chrom, '.vcf']) radia_log = ''.join([work_dir, '/radia_', chrom, '_radia.log']) @@ -214,7 +214,7 @@ def run_filter_radia(job, bams, radia_file, univ_options, radia_options, chrom): for key in ('cosmic_beds', 'dbsnp_beds', 'retrogene_beds', 'pseudogene_beds', 'gencode_beds'): input_files[key] = untargz(input_files[key], work_dir) - input_files = {key: docker_path(path) for key, path in input_files.items()} + input_files = {key: docker_path(path) for key, path in list(input_files.items())} filterradia_log = ''.join([work_dir, '/radia_filtered_', chrom, '_radia.log']) parameters = [univ_options['patient'], # shortID diff --git a/src/protect/mutation_calling/somaticsniper.py b/src/protect/mutation_calling/somaticsniper.py index ae148f04..42825439 100644 --- a/src/protect/mutation_calling/somaticsniper.py +++ b/src/protect/mutation_calling/somaticsniper.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function + from collections import defaultdict from math import ceil @@ -146,7 +146,7 @@ def run_somaticsniper_full(job, tumor_bam, normal_bam, univ_options, somaticsnip for key in ('genome.fa', 'genome.fa.fai'): input_files[key] = untargz(input_files[key + '.tar.gz'], work_dir) - input_files = {key: docker_path(path) for key, path in input_files.items()} + input_files = {key: docker_path(path) for key, path in list(input_files.items())} output_file = os.path.join(work_dir, 'somatic-sniper_full.vcf') parameters = ['-f', input_files['genome.fa'], @@ -190,7 +190,7 @@ def filter_somaticsniper(job, tumor_bam, somaticsniper_output, tumor_pileup, uni for key in ('genome.fa', 'genome.fa.fai'): input_files[key] = untargz(input_files[key + '.tar.gz'], work_dir) - input_files = {key: docker_path(path) for key, path in input_files.items()} + input_files = {key: docker_path(path) for key, path in list(input_files.items())} # Run snpfilter.pl parameters = ['snpfilter.pl', @@ -278,7 +278,7 @@ def run_pileup(job, tumor_bam, univ_options, somaticsniper_options): for key in ('genome.fa', 'genome.fa.fai'): input_files[key] = untargz(input_files[key + '.tar.gz'], work_dir) - input_files = {key: docker_path(path) for key, path in input_files.items()} + input_files = {key: docker_path(path) for key, path in list(input_files.items())} parameters = ['pileup', '-cvi', diff --git a/src/protect/mutation_calling/strelka.py b/src/protect/mutation_calling/strelka.py index 0a1a0e7b..172e4691 100644 --- a/src/protect/mutation_calling/strelka.py +++ b/src/protect/mutation_calling/strelka.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function + from math import ceil from protect.common import (docker_call, @@ -130,7 +130,7 @@ def run_strelka_full(job, tumor_bam, normal_bam, univ_options, strelka_options): for key in ('genome.fa', 'genome.fa.fai', 'config.ini'): input_files[key] = untargz(input_files[key + '.tar.gz'], work_dir) - input_files = {key: docker_path(path) for key, path in input_files.items()} + input_files = {key: docker_path(path) for key, path in list(input_files.items())} parameters = [input_files['config.ini'], input_files['tumor.bam'], diff --git a/src/protect/mutation_translation.py b/src/protect/mutation_translation.py index 68e76221..21f3ee1e 100644 --- a/src/protect/mutation_translation.py +++ b/src/protect/mutation_translation.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import, print_function + from collections import defaultdict from math import ceil @@ -83,12 +83,12 @@ def run_transgene(job, snpeffed_file, rna_bam, univ_options, transgene_options, input_files['pepts.fa'] = untargz(input_files['pepts.fa.tar.gz'], work_dir) input_files['genome.fa'] = untargz(input_files['genome.fa.tar.gz'], work_dir) input_files['annotation.gtf'] = untargz(input_files['annotation.gtf.tar.gz'], work_dir) - input_files = {key: docker_path(path) for key, path in input_files.items()} + input_files = {key: docker_path(path) for key, path in list(input_files.items())} parameters = ['--peptides', input_files['pepts.fa'], '--prefix', 'transgened', '--pep_lens', '9,10,15', - '--cores', str(transgene_options['n']), + '--cores', str(20), '--genome', input_files['genome.fa'], '--annotation', input_files['annotation.gtf']] @@ -107,7 +107,7 @@ def run_transgene(job, snpeffed_file, rna_bam, univ_options, transgene_options, fusion_files = get_files_from_filestore(job, fusion_files, work_dir, docker=False) fusion_files['transcripts.fa'] = untargz(fusion_files['transcripts.fa.tar.gz'], work_dir) - fusion_files = {key: docker_path(path) for key, path in fusion_files.items()} + fusion_files = {key: docker_path(path) for key, path in list(fusion_files.items())} parameters += ['--transcripts', fusion_files['transcripts.fa'], '--fusions', fusion_files['fusion_calls']] diff --git a/src/protect/pipeline/ProTECT.py b/src/protect/pipeline/ProTECT.py index 6b40fb79..c8e14745 100644 --- a/src/protect/pipeline/ProTECT.py +++ b/src/protect/pipeline/ProTECT.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -21,7 +21,7 @@ Program info can be found in the docstring of the main function. Details can also be obtained by running the script with -h . """ -from __future__ import print_function + from collections import defaultdict from multiprocessing import cpu_count @@ -160,7 +160,7 @@ def _add_default_entries(input_dict, defaults_dict): :return: updated dict :rtype: dict """ - for key, value in defaults_dict.iteritems(): + for key, value in defaults_dict.items(): if key == 'patients': print('Cannot default `patients`.') continue @@ -302,8 +302,8 @@ def parse_patients(job, patient_dict, skip_fusions=False): if f + '_fastq_2' not in output_dict: output_dict[f + '_fastq_2'] = get_fastq_2(job, patient_dict['patient_id'], f, output_dict[f + '_fastq_1']) - output_dict['gdc_inputs'] = [k for k, v in output_dict.items() if str(v).startswith('gdc')] - if not any('dna' in k for k in output_dict.keys()): + output_dict['gdc_inputs'] = [k for k, v in list(output_dict.items()) if str(v).startswith('gdc')] + if not any('dna' in k for k in list(output_dict.keys())): # There are no input DNA files so we cannot filter for oxog output_dict['filter_for_OxoG'] = False return output_dict @@ -354,7 +354,7 @@ def _parse_config_file(job, config_file, max_cores=None): # Flags to check for presence of encryption keys if required gdc_inputs = ssec_encrypted = False - for key in input_config.keys(): + for key in list(input_config.keys()): if key == 'patients': # Ensure each patient contains the required entries for sample_name in input_config[key]: @@ -410,7 +410,7 @@ def _parse_config_file(job, config_file, max_cores=None): if key == 'alignment': append_subgroup = ['post'] elif key == 'mutation_calling': - mutation_caller_list = input_config[key].keys() + mutation_caller_list = list(input_config[key].keys()) append_subgroup = [] else: append_subgroup = [] @@ -443,7 +443,7 @@ def parse_config_file(job, config_file, max_cores=None): sample_set, univ_options, processed_tool_inputs = _parse_config_file(job, config_file, max_cores) # Start a job for each sample in the sample set - for patient_id in sample_set.keys(): + for patient_id in list(sample_set.keys()): job.addFollowOnJobFn(launch_protect, sample_set[patient_id], univ_options, processed_tool_inputs) return None @@ -513,7 +513,7 @@ def launch_protect(job, patient_data, univ_options, tool_options): haplotype_patient = job.wrapJobFn(get_patient_mhc_haplotype, sample_prep.rv()) sample_prep.addChild(haplotype_patient) else: - assert None not in fastq_files.values() + assert None not in list(fastq_files.values()) # We are guaranteed to have fastqs here for sample_type in 'tumor_dna', 'normal_dna', 'tumor_rna': phlat_files[sample_type] = job.wrapJobFn( @@ -633,7 +633,7 @@ def launch_protect(job, patient_data, univ_options, tool_options): # Fusions have been handled above, and we don't need to align DNA get_mutations = None else: - assert (None, None) not in zip(fastq_files.values(), bam_files.values()) + assert (None, None) not in list(zip(list(fastq_files.values()), list(bam_files.values()))) for sample_type in 'tumor_dna', 'normal_dna': if bam_files[sample_type] is None: assert fastq_files[sample_type] is not None @@ -677,7 +677,7 @@ def launch_protect(job, patient_data, univ_options, tool_options): bam_files[sample_type].addChild(mutations[caller]) bam_files['tumor_rna'].addChild(mutations['radia']) get_mutations = job.wrapJobFn(run_mutation_aggregator, - {caller: cjob.rv() for caller, cjob in mutations.items()}, + {caller: cjob.rv() for caller, cjob in list(mutations.items())}, univ_options, disk='100M', memory='100M', cores=1).encapsulate() for caller in mutations: @@ -687,19 +687,21 @@ def launch_protect(job, patient_data, univ_options, tool_options): # We may need the tumor one depending on OxoG if not patient_data['filter_for_OxoG']: get_mutations.addChild(delete_bam_files['tumor_dna']) - + print("DEBUG: got to where snpeff should run") if get_mutations is not None: snpeff = job.wrapJobFn(run_snpeff, get_mutations.rv(), univ_options, tool_options['snpeff'], disk=PromisedRequirement(snpeff_disk, tool_options['snpeff']['index'])) get_mutations.addChild(snpeff) else: + print("DEBUG: no snpeff") snpeff = None # The rest of the subgraph should be unchanged tumor_dna_bam = bam_files['tumor_dna'].rv() if patient_data['filter_for_OxoG'] else None fusion_calls = fusions.rv() if fusions else None snpeffed_calls = snpeff.rv() if snpeff else None + print("DEBUG: got to transgene wrap") transgene = job.wrapJobFn(run_transgene, snpeffed_calls, bam_files['tumor_rna'].rv(), univ_options, tool_options['transgene'], disk=PromisedRequirement(transgene_disk, bam_files['tumor_rna'].rv()), @@ -714,24 +716,25 @@ def launch_protect(job, patient_data, univ_options, tool_options): transgene.addChild(delete_bam_files['tumor_dna']) if fusions: fusions.addChild(transgene) - + print("DEBUG: got to spawn mhc wrap") spawn_mhc = job.wrapJobFn(spawn_antigen_predictors, transgene.rv(), haplotype_patient.rv(), univ_options, (tool_options['mhci'], tool_options['mhcii']), disk='100M', memory='100M', cores=1).encapsulate() haplotype_patient.addChild(spawn_mhc) transgene.addChild(spawn_mhc) - + print("DEBUG: goat to merge mhc wrap") merge_mhc = job.wrapJobFn(merge_mhc_peptide_calls, spawn_mhc.rv(), transgene.rv(), univ_options, disk='100M', memory='100M', cores=1) spawn_mhc.addFollowOn(merge_mhc) transgene.addChild(merge_mhc) - + print("DEBUG: got to rankboost wrap") rankboost = job.wrapJobFn(wrap_rankboost, rsem.rv(), merge_mhc.rv(), transgene.rv(), univ_options, tool_options['rankboost'], disk='100M', memory='100M', cores=1) rsem.addChild(rankboost) merge_mhc.addChild(rankboost) transgene.addChild(rankboost) + print("DEBUG: report success") report_success = job.wrapJobFn(email_report, univ_options) rankboost.addChild(report_success) return None diff --git a/src/protect/pipeline/__init__.py b/src/protect/pipeline/__init__.py index ba9b9130..dabfb0af 100644 --- a/src/protect/pipeline/__init__.py +++ b/src/protect/pipeline/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,4 +13,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + diff --git a/src/protect/qc/__init__.py b/src/protect/qc/__init__.py index ba9b9130..dabfb0af 100644 --- a/src/protect/qc/__init__.py +++ b/src/protect/qc/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,4 +13,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + diff --git a/src/protect/qc/rna.py b/src/protect/qc/rna.py index 42334920..37b8ad99 100644 --- a/src/protect/qc/rna.py +++ b/src/protect/qc/rna.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function + from math import ceil from protect.common import docker_call, docker_path, get_files_from_filestore, is_gzipfile @@ -47,7 +47,7 @@ def run_cutadapt(job, fastqs, univ_options, cutadapt_options): for read_file in 'rna_1.fastq', 'rna_2.fastq': os.symlink(read_file, read_file + gz) input_files[read_file + gz] = input_files[read_file] + gz - input_files = {key: docker_path(path) for key, path in input_files.items()} + input_files = {key: docker_path(path) for key, path in list(input_files.items())} parameters = ['-a', cutadapt_options['a'], # Fwd read 3' adapter '-A', cutadapt_options['A'], # Rev read 3' adapter '-m', '35', # Minimum size of read diff --git a/src/protect/rankboost.py b/src/protect/rankboost.py index bd9bf514..2c0b0a53 100644 --- a/src/protect/rankboost.py +++ b/src/protect/rankboost.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # diff --git a/src/protect/test/__init__.py b/src/protect/test/__init__.py index 382c581b..219056ee 100644 --- a/src/protect/test/__init__.py +++ b/src/protect/test/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -16,7 +16,7 @@ # A lot of this code was taken from toil/test/src/__init__.py -from __future__ import absolute_import + import logging import os import tempfile diff --git a/src/protect/test/ci/__init__.py b/src/protect/test/ci/__init__.py index ba9b9130..dabfb0af 100644 --- a/src/protect/test/ci/__init__.py +++ b/src/protect/test/ci/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,4 +13,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + diff --git a/src/protect/test/ci/test_protect.py b/src/protect/test/ci/test_protect.py index 2fb3eae8..906160dc 100644 --- a/src/protect/test/ci/test_protect.py +++ b/src/protect/test/ci/test_protect.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -19,7 +19,7 @@ Affiliation : UCSC BME, UCSC Genomics Institute File : protect/test/test_reporting.py """ -from __future__ import print_function + from protect.test import ProtectTest @@ -173,8 +173,8 @@ def _test_ran_successfully(self, expected_dirs): expected_contents = {} for dir in expected_dirs: if isinstance(dir, dict): - assert len(dir.keys()) == 1 - if dir.keys()[0] == 'mutations': + assert len(list(dir.keys())) == 1 + if list(dir.keys())[0] == 'mutations': expected_contents['mutations'] = ('/mnt/ephemeral/done/TEST/mutations', sorted(dir['mutations']), []) @@ -198,7 +198,7 @@ def _test_ran_successfully(self, expected_dirs): else: expected_contents['mutations_' + caller] = \ contents_per_dir['mutations'][caller] - elif dir.keys()[0] == 'alignments': + elif list(dir.keys())[0] == 'alignments': alignment_files = [] for tissue_type in dir['alignments']: alignment_files.extend(contents_per_dir['alignments'][tissue_type]) @@ -211,7 +211,7 @@ def _test_ran_successfully(self, expected_dirs): expected_contents[dir] = contents_per_dir[dir] expected_outputs = [('/mnt/ephemeral/done/TEST', - sorted([x for x in expected_contents.keys() + sorted([x for x in list(expected_contents.keys()) if not x.startswith('mutations_')]), [])] expected_outputs.extend([expected_contents[d] for d in sorted(expected_contents.keys())]) diff --git a/src/protect/test/unit/__init__.py b/src/protect/test/unit/__init__.py index ba9b9130..dabfb0af 100644 --- a/src/protect/test/unit/__init__.py +++ b/src/protect/test/unit/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -13,4 +13,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import + diff --git a/src/protect/test/unit/test_alignments.py b/src/protect/test/unit/test_alignments.py index a8d2d3a8..0ea387eb 100644 --- a/src/protect/test/unit/test_alignments.py +++ b/src/protect/test/unit/test_alignments.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -19,7 +19,7 @@ Affiliation : UCSC BME, UCSC Genomics Institute File : protect/test/test_file_downloads.py """ -from __future__ import print_function + import os import subprocess @@ -75,7 +75,7 @@ def _get_test_bwa_files(job): :return: FSID for the rsem file """ - base_call = 's3am download s3://cgl-pipeline-inputs/protect/ci_references/' + base_call = 's3am download S3://protect-data/hg38_references/' subprocess.check_call((base_call + 'Tum_1.fq.gz Tum_1.fq.gz').split(' ')) subprocess.check_call((base_call + 'Tum_2.fq.gz Tum_2.fq.gz').split(' ')) return [job.fileStore.writeGlobalFile('Tum_1.fq.gz'), @@ -105,7 +105,7 @@ def _get_test_star_files(job): :return: FSID for the rsem file """ - base_call = 's3am download s3://cgl-pipeline-inputs/protect/ci_references/' + base_call = 's3am download S3://protect-data/hg38_references/' subprocess.check_call((base_call + 'Rna_1.fq.gz Rna_1.fq.gz').split(' ')) subprocess.check_call((base_call + 'Rna_2.fq.gz Rna_2.fq.gz').split(' ')) return [job.fileStore.writeGlobalFile('Rna_1.fq.gz'), diff --git a/src/protect/test/unit/test_file_downloads.py b/src/protect/test/unit/test_file_downloads.py index 35994668..39f115a2 100644 --- a/src/protect/test/unit/test_file_downloads.py +++ b/src/protect/test/unit/test_file_downloads.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -19,7 +19,7 @@ Affiliation : UCSC BME, UCSC Genomics Institute File : protect/test/test_file_downloads.py """ -from __future__ import print_function + from protect.common import get_file_from_s3 from protect.common import get_file_from_url diff --git a/src/protect/test/unit/test_mutation_callers.py b/src/protect/test/unit/test_mutation_callers.py index efa5d3e7..c30702da 100644 --- a/src/protect/test/unit/test_mutation_callers.py +++ b/src/protect/test/unit/test_mutation_callers.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -19,7 +19,7 @@ Affiliation : UCSC BME, UCSC Genomics Institute File : protect/test/test_file_downloads.py """ -from __future__ import print_function + from protect.common import untargz from protect.mutation_calling.muse import run_muse from protect.mutation_calling.mutect import run_mutect @@ -101,7 +101,7 @@ def test_star_fusion(self): def _get_fusion_options(job): star_fusion_options = {} fusion_inspector_options = {} - call = 's3am download S3://cgl-pipeline-inputs/protect/ci_references/ci_star_fusion_compatible_index.tar.gz ./index.tar.gz' + call = 's3am download S3://protect-data/hg38_references/ci_star_fusion_compatible_index.tar.gz ./index.tar.gz' subprocess.check_call(call.split(' ')) star_fusion_options['index'] = fusion_inspector_options['index'] = job.fileStore.writeGlobalFile('index.tar.gz') # Trinity now sets a minimum for the number of reads, so don't run it @@ -192,7 +192,7 @@ def _get_test_fusion_reads(job): :return: FSID for each paired FASTQ """ - base_call = 's3am download S3://cgl-pipeline-inputs/protect/ci_references/' + base_call = 's3am download S3://protect-data/hg38_references/' samples = ['RNA_CD74_ROS1_1.fq.gz', 'RNA_CD74_ROS1_2.fq.gz'] for sample in samples: call = '{base}{sample} ./{sample}'.format(base=base_call, sample=sample) @@ -209,7 +209,7 @@ def _get_test_fusion_junction(job): :return: FSID for each paired FASTQ """ - base_call = 's3am download S3://cgl-pipeline-inputs/protect/ci_references/' + base_call = 's3am download S3://protect-data/hg38_references/' sample = 'CD74_ROS1_Chimeric.out.junction' call = '{base}{sample} ./ChimericJunction'.format(base=base_call, sample=sample) subprocess.check_call(call.split(' ')) diff --git a/src/protect/test/unit/test_rankboost.py b/src/protect/test/unit/test_rankboost.py index 0c271f19..442ae22b 100644 --- a/src/protect/test/unit/test_rankboost.py +++ b/src/protect/test/unit/test_rankboost.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -19,7 +19,7 @@ Affiliation : UCSC BME, UCSC Genomics Institute File : protect/test/test_rankboost.py """ -from __future__ import print_function + from protect.pipeline.ProTECT import _parse_config_file from protect.rankboost import wrap_rankboost diff --git a/src/protect/test/unit/test_reporting.py b/src/protect/test/unit/test_reporting.py index 9c17fc91..00572421 100644 --- a/src/protect/test/unit/test_reporting.py +++ b/src/protect/test/unit/test_reporting.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -19,7 +19,7 @@ Affiliation : UCSC BME, UCSC Genomics Institute File : protect/test/test_reporting.py """ -from __future__ import print_function + import os diff --git a/src/protect/test/unit/test_snpeff.py b/src/protect/test/unit/test_snpeff.py index 39381c55..c016123d 100644 --- a/src/protect/test/unit/test_snpeff.py +++ b/src/protect/test/unit/test_snpeff.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -19,7 +19,7 @@ Affiliation : UCSC BME, UCSC Genomics Institute File : protect/test/test_snpeff.py """ -from __future__ import print_function + from protect.mutation_annotation.snpeff import run_snpeff from protect.pipeline.ProTECT import _parse_config_file diff --git a/src/protect/test/unit/test_spawn_antigen_predictors.py b/src/protect/test/unit/test_spawn_antigen_predictors.py index f00bb393..db3bfa5c 100644 --- a/src/protect/test/unit/test_spawn_antigen_predictors.py +++ b/src/protect/test/unit/test_spawn_antigen_predictors.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # @@ -19,7 +19,7 @@ Affiliation : UCSC BME, UCSC Genomics Institute File : protect/test/test_spawn_antigen_predictors.py """ -from __future__ import print_function + from protect.binding_prediction.common import spawn_antigen_predictors, merge_mhc_peptide_calls from protect.pipeline.ProTECT import _parse_config_file diff --git a/src/protect/test/unit/test_transgene b/src/protect/test/unit/test_transgene index 8adfeaec..5a7ac91e 100644 --- a/src/protect/test/unit/test_transgene +++ b/src/protect/test/unit/test_transgene @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao # diff --git a/src/protect/version.py b/src/protect/version.py index f0db84aa..b42ec478 100644 --- a/src/protect/version.py +++ b/src/protect/version.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2016 UCSC Computational Genomics Lab # Original contributor: Arjun Arkal Rao #