Skip to content

Commit

Permalink
fix erroneous messages printed about pre-existing files after run fin…
Browse files Browse the repository at this point in the history
…ished
  • Loading branch information
ssadedin committed Apr 23, 2019
1 parent f86987e commit 62b381a
Show file tree
Hide file tree
Showing 7 changed files with 78 additions and 10 deletions.
2 changes: 1 addition & 1 deletion src/main/groovy/bpipe/Dependencies.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ class Dependencies {
/**
* List of output files created in *this run*
*/
List<String> outputFilesGenerated = []
List<PipelineFile> outputFilesGenerated = []

/**
* List of files whose timestamps are overridden, so that their real
Expand Down
34 changes: 25 additions & 9 deletions src/main/groovy/bpipe/Pipeline.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -1621,20 +1621,13 @@ public class Pipeline implements ResourceRequestor {

Dependencies.instance.reset()
def graph = Dependencies.instance.outputGraph
List all = Dependencies.instance.findLeaves(graph)*.values.flatten()*.outputPath
List<PipelineFile> leaves = Dependencies.instance.findLeaves(graph)*.values.flatten()*.outputFile

Utils.time("Save output graph") {
Dependencies.instance.saveOutputGraphCache()
}

def runFiles = Dependencies.instance.outputFilesGenerated

// Sort to put files generated by this run first
all = all.sort {runFiles.contains(it) ? 0 : 1 }
// Remove files that don't exist any more
.grep { it && new File(it).exists() }.
// Add 'pre-existing' for files that were not generated by this Bpipe run
collect { runFiles.contains(it) ? it : it + ' (pre-existing)' }
List<String> all = formatOutputFiles(leaves)

if(all.size() == 1) {
log.info "Output is " + all[0]
Expand All @@ -1655,6 +1648,29 @@ public class Pipeline implements ResourceRequestor {
}
rootContext.outputLog.flush()
}

@CompileStatic
private List<String> formatOutputFiles(List<PipelineFile> all) {

final Map<String, PipelineFile> runFiles = Dependencies.theInstance.outputFilesGenerated.collectEntries {
[ it.toString(), it]
}

List<String> result = (List<String>)all

// Sort to put files generated by this run first
.sort { runFiles.containsKey(it?.toString()) ? 0 : 1 }

// Remove files that don't exist any more
.grep { PipelineFile pf -> pf && pf.exists() }

// Add 'pre-existing' for files that were not generated by this Bpipe run
.collect {
runFiles.containsKey(it.toString()) ? it : (it.toString() + ' (pre-existing)')
}

return result
}

/**
* Convenience function to convert the given string to a file.
Expand Down
1 change: 1 addition & 0 deletions tests/print_created_files/cleanup.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
rm -rf .bpipe test.*.csv test.*.xml
22 changes: 22 additions & 0 deletions tests/print_created_files/run.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
source ../testsupport.sh

set -e

bpipe run test2.groovy test.txt > test.out

grep -q 'Output is test.hello.csv' test.out || err "did not print expected message about output"
grep -q 'pre-existing' test.out && err "printed output as pre-existing when not"

bpipe run -p worldy=true test2.groovy test.txt > test2.out

grep -q 'Outputs are:' test2.out || err "did not print expected message about output"
grep -q 'test.world.xml' test2.out || err "did not print expected message about output file"
grep -q 'xml.*pre-existing' test2.out && err "printed output as pre-existing when not"
grep -q 'csv.*pre-existing' test2.out || err "did not print CSV outout as pre-existing when it was"

true





Empty file.
9 changes: 9 additions & 0 deletions tests/print_created_files/test1.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
hello = {
exec """
cp $input.txt $output.xml
"""
}

run {
hello
}
20 changes: 20 additions & 0 deletions tests/print_created_files/test2.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
hello = {
exec """
cp $input.txt $output.csv
"""
}

world = {
var worldy: false

if(worldy) {
exec """
cp $input.txt $output.xml
"""
}
}

run {
hello + world
}

0 comments on commit 62b381a

Please sign in to comment.