Skip to content

Commit

Permalink
Fixed incorrect number of outputs mentioned in error message in some …
Browse files Browse the repository at this point in the history
…cases

Fixed referencing outputs[2] or greater failing
Made tolerant of a range of procs passed in config (procs="1-5") for resource managers that support that
  • Loading branch information
ssadedin committed Nov 9, 2013
1 parent d4527af commit 58bb08c
Show file tree
Hide file tree
Showing 10 changed files with 62 additions and 9 deletions.
6 changes: 3 additions & 3 deletions src/main/groovy/bpipe/PipelineCategory.groovy
Expand Up @@ -216,7 +216,7 @@ class PipelineCategory {
// If the filterInputs option is set, match input files on the region name
def childInputs = input

def filterInputs = chr.config?.filterInputs
def filterInputs = chr instanceof Chr ? chr.config?.filterInputs : "auto"

if(filterInputs == "auto") {
filterInputs = Utils.box(input).any { it.matches(/.*\.chr[1-9A-Z_]*\..*$/) }
Expand All @@ -243,10 +243,10 @@ class PipelineCategory {

log.info "Adding dummy prior stage for thread ${Thread.currentThread().id} with outputs : $dummyPriorContext.output"
child.addStage(dummyPriorStage)
def region = chr.region
def region = chr instanceof Chr ? chr.region : ""
child.variables += [chr: region]
child.variables += [region: region]
child.name = chr.name
child.name = chr instanceof Chr ? chr.name : chr
child.runSegment(childInputs, segmentClosure)
}
catch(Exception e) {
Expand Down
13 changes: 11 additions & 2 deletions src/main/groovy/bpipe/PipelineContext.groovy
Expand Up @@ -405,7 +405,11 @@ class PipelineContext {
}

def getOutputs() {
return Utils.box(getOutput())
def raw = getOutput()
// Used to 'box' the result, but now this is a PipelineOutput that supports direct access by index,
// so should not need it
// def result = Utils.box(raw)
return raw
}

def getOutputByIndex(int index) {
Expand Down Expand Up @@ -1408,8 +1412,13 @@ class PipelineContext {
// which may get given a crazy high number of threads
if(config) {
def procs = command.getConfig(Utils.box(this.input)).procs
if(procs)
if(procs) {
if(procs instanceof String) {
// Allow range of integers
procs = procs.replaceAll(" *-.*\$","")
}
actualThreads = String.valueOf(Math.min(procs.toInteger(), actualThreads.toInteger()))
}
}

command.command = joined.replaceAll(THREAD_LAZY_VALUE, actualThreads)
Expand Down
8 changes: 4 additions & 4 deletions src/main/groovy/bpipe/PipelineOutput.groovy
Expand Up @@ -119,10 +119,10 @@ class PipelineOutput {
* exec "cp foo.txt ${output[0]}"
*/
String getAt(int i) {
def inputs = Utils.box(this.output)
if(inputs.size() <= i)
throw new PipelineError("Insufficient outputs: output $i was referenced but there are only ${output.size()} outputs to choose from")
return inputs[i]
def outs = Utils.box(this.output)
if(outs.size() <= i)
throw new PipelineError("Insufficient outputs: output ${i+1} was referenced but there are only ${outs.size()} outputs to choose from")
return outs[i]
}

/**
Expand Down
9 changes: 9 additions & 0 deletions tests/custom_chr/run.sh
@@ -0,0 +1,9 @@
source ../testsupport.sh

run

grep -q "^mars" test.out || err "Failed to find mars printed out"
grep -q "^world" test.out || err "Failed to find world printed out"
grep -q "^jupiter" test.out || err "Failed to find mars printed out"

true
10 changes: 10 additions & 0 deletions tests/custom_chr/test.groovy
@@ -0,0 +1,10 @@
hello = {
exec "echo $branch"
}

parallel_paths = ["mars","world","jupiter"] as Set

run {
parallel_paths * [ hello ]
}

1 change: 1 addition & 0 deletions tests/multi_computed/$
@@ -0,0 +1 @@
mars
Binary file added tests/multi_computed/.test.groovy.swp
Binary file not shown.
1 change: 1 addition & 0 deletions tests/multi_computed/cleanup.sh
@@ -0,0 +1 @@
rm -f e g null out*.txt hello*
6 changes: 6 additions & 0 deletions tests/multi_computed/run.sh
@@ -0,0 +1,6 @@
source ../testsupport.sh

run

exists out1.txt out2.txt out3.txt

17 changes: 17 additions & 0 deletions tests/multi_computed/test.groovy
@@ -0,0 +1,17 @@
/* The example from the documentation */

hello = {
// Build a list of commands to execute in parallel
def outputs = (1..3).collect { "out${it}.txt" }

// Compute the commands we are going to execute
int n = 0
def commands =["mars","jupiter","earth"].collect{"echo $it > ${outputs[n++]}"}

// Tell Bpipe to produce the outputs from the commands
produce(outputs) {
multiExec commands
}
}

run { hello }

0 comments on commit 58bb08c

Please sign in to comment.