Workarounds for non-existent but declared Bash array bug. Some correc…

…tions after review.

README.md

@@ -12,7 +12,7 @@ Note that after the swapping, the read-numbers are reversed. What was R1 in the
 
 Furthermore, not all reads can be unambiguously classified. These unclassified reads are currently dropped. 
 
-The original script can be found [here](https://github.com/cimbusch/TWGBS.git).
+The original script with a documentation of the underlying ideas can be found [here](https://github.com/cimbusch/TWGBS.git).
 
 ## Change Logs
 

resources/analysisTools/bisulfiteWorkflow/WGBS_read_pair_reconstruction.py

@@ -3,6 +3,7 @@
 # Copyright (c) 2018 German Cancer Research Center (DKFZ).
 #
 # Distributed under the MIT License (license terms are at https://github.com/cimbusch/TWGBS).
+# Commit: a202ae5d9b19c46b348c58523a17a4a7c66d116f
 #
 """
 Code to reconstruct correct R1-R2 reads relation from base ratio:

resources/analysisTools/bisulfiteWorkflow/bwaMemSortSlimWithReadConversionForBisulfiteData.sh

@@ -100,11 +100,11 @@ if [[ "$bamFileExists" == "false" ]]; then	# we have to make the BAM
     getFastqAsciiStream "$RAW_SEQ_2" > $(getPairedPipeEndPath 2 "$fqName") & procUnpack2=$!
 
     if [[ "$qualityScore" == "illumina" ]]; then
-	    extendPipe $(mkPairedPipeName 1 "$fqName") "qScore" -- toIlluminaScore
-		extendPipe $(mkPairedPipeName 2 "$fqName") "qScore" -- toIlluminaScore
+        extendPipe $(mkPairedPipeName 1 "$fqName") "qScore" -- toIlluminaScore
+        extendPipe $(mkPairedPipeName 2 "$fqName") "qScore" -- toIlluminaScore
     fi
 
-    if [[ "$useAdaptorTrimming:-false" == "true" ]]; then
+    if [[ "${useAdaptorTrimming:-false}" == "true" ]]; then
         extendPipePair "$fqName" "trimmomatic" -- trimmomatic
     fi
 
@@ -209,6 +209,7 @@ else	# make sure to rename BAM file when it has been produced correctly
 	fi
 fi
 
+waitForRegisteredPids_BashSucksVersion
 wait $procUnpack1 || throw 39 "Error from reading FASTQ 1"
 wait $procUnpack1 || throw 40 "Error from reading FASTQ 2"
 

resources/analysisTools/qcPipeline/bashLib.sh

@@ -178,6 +178,7 @@ waitForRegisteredPids_BashSucksVersion() {
     jobs
     declare -a realPids=$(listPids)
     if [[ -v realPids && ${#realPids[@]} -gt 0 ]]; then
+        # TODO Make this a look and report the exact pid that failed (or the key, after switching from array to dictionary).
         wait ${realPids[@]}
         declare EXIT_CODE=$?
         if [[ ${EXIT_CODE} -ne 0 ]]; then
@@ -213,6 +214,10 @@ cleanUp_BashSucksVersion() {
 waitForRegisteredPids() {
     jobs
     wait ${pids[@]}
+    declare EXIT_CODE=$?
+    if [[ ${EXIT_CODE} -ne 0 ]]; then
+        throw ${EXIT_CODE} "One of the following processes ended with exit code ${EXIT_CODE}: ${pids[@]}"
+    fi
     pids=()
 }
 setUp() {
@@ -251,6 +256,7 @@ cleanUp() {
 _pipePath="$RODDY_SCRATCH"
 
 # Maintain a mapping of pipe-basenames to current pipe-ends.
+# Please read https://www.artificialworlds.net/blog/2012/10/17/bash-associative-array-examples/ to understand the associative array syntax below.
 initPipeEnds() {
     unset _pipeEnds
     declare -Ag _pipeEnds=()
@@ -269,6 +275,7 @@ mkPipePath() {
 }
 
 # Just get the path from the registry.
+# Please read https://www.artificialworlds.net/blog/2012/10/17/bash-associative-array-examples/ to understand the associative array syntax below.
 getPipeEndPath() {
     local pipeName="${1:?No pipename}"
     if [[ ! ${_pipeEnds[$pipeName]+_} ]]; then
@@ -300,6 +307,7 @@ updatePipeEndPath() {
 }
 
 # Create a pipe and register it as new pipe-end in the registry. It will be the source of pipe.
+# Please read https://www.artificialworlds.net/blog/2012/10/17/bash-associative-array-examples/ to understand the associative array syntax below.
 mkPipeSource() {
     local pipeName="${1:?No pipename}"
     if [[ ${_pipeEnds[$pipeName]+_} ]]; then
@@ -325,13 +333,19 @@ extendPipe() {
     fi
     local command="${1:?No command/function}"
     shift
-    declare -a args=("$@")
+    declare -a rest=("$@")
 
     local inpipe=$(getPipeEndPath "$pipeName")
     updatePipeEndPath "$pipeName" "$tag"
     local outpipe=$(getPipeEndPath "$pipeName")
 
-    "$command" "$inpipe" "$outpipe" "${args[@]}" & registerPid
+    # This &#"@! is because Bash SUCKS! Empty arrays do not exist Bash < 4.4
+    if [[ -v rest ]]; then
+        "$command" "$inpipe" "$outpipe" "${rest[@]}" & registerPid
+    else
+        "$command" "$inpipe" "$outpipe" & registerPid
+    fi
+
 }
 
 

resources/analysisTools/qcPipeline/workflowLib.sh

@@ -33,7 +33,7 @@ mbuf () {
     local bufferSize="$1"
     shift
     assertNonEmpty "$bufferSize" "No buffer size defined for mbuf()" || return $?
-    "$MBUFFER_BINARY" -m "$bufferSize" -q -l /dev/null ${@}
+    "$MBUFFER_BINARY" -m "$bufferSize" -q -l /dev/null "$@"
 }
 
 
@@ -190,7 +190,7 @@ extendPipePair() {
     fi
     local command="${1:?No command/function}"
     shift
-    local args=("$@")
+    declare -a rest=("$@")
 
     local pipe1Name=$(mkPairedPipeName 1 "$pipeName")
     local pipe2Name=$(mkPairedPipeName 2 "$pipeName")
@@ -203,7 +203,12 @@ extendPipePair() {
     updatePipeEndPath "$pipe2Name" "$tag"
     local r2_outpipe=$(getPipeEndPath "$pipe2Name")
 
-    "$command" "$r1_inpipe" "$r2_inpipe" "$r1_outpipe" "$r2_outpipe" "${args[@]}" & registerPid
+    # This &#"@! is because Bash SUCKS! Empty arrays do not exist Bash < 4.4
+    if [[ -v rest ]]; then
+        "$command" "$r1_inpipe" "$r2_inpipe" "$r1_outpipe" "$r2_outpipe" "${rest[@]}" & registerPid
+    else
+        "$command" "$r1_inpipe" "$r2_inpipe" "$r1_outpipe" "$r2_outpipe" & registerPid
+    fi
 }
 
 
@@ -245,15 +250,15 @@ methylCfqconv() {
 }
 
 trimmomatic() {
-    local i1="${1:?No R1 input}"
-    local i2="${2:?No R2 input}"
-    local o1="${3:?No R1 output}"
-    local o2="${4:?No R2 output}"
+    local input1="${1:?No R1 input}"
+    local input2="${2:?No R2 input}"
+    local output1="${3:?No R1 output}"
+    local output2="${4:?No R2 output}"
 
     local u1=/dev/null
     local u2=/dev/null
 
-    "$TRIMMOMATIC_BINARY" "$ADAPTOR_TRIMMING_OPTIONS_0" "$i1" "$i2" "$o1" "$u1" "$o2" "$u2" $ADAPTOR_TRIMMING_OPTIONS_1
+    "$TRIMMOMATIC_BINARY" $ADAPTOR_TRIMMING_OPTIONS_0 "$input1" "$input2" "$output1" "$u1" "$output2" "$u2" $ADAPTOR_TRIMMING_OPTIONS_1
 }
 
 

resources/tests/analysisTools/qcPipeline/bashLib.sh

@@ -48,7 +48,8 @@ teardownPipePath() {
 
 testSetGetPipeEndPath() {
     setupPipePath
-    assertFalse "test getPipeEndPath 'doesnotexist'"
+    test getPipeEndPath 'doesnotexist'
+    assertFalse "Error exit on non-existent pipe-name access" $?
     setPipeEndPath "test" "/a/b/c"
     assertEquals "/a/b/c" "$(getPipeEndPath 'test')"
     teardownPipePath