Skip to content
Permalink
Browse files

support to template dirs created by bpipe agent with incrementing values

  • Loading branch information...
ssadedin committed May 22, 2019
1 parent 5abbf93 commit d1a3aa50469cfaebd989f0a5f499a9b63fe4d4fb
@@ -28,6 +28,8 @@ import bpipe.ExecutedProcess
import bpipe.Utils
import groovy.transform.CompileStatic
import groovy.util.logging.Log
import java.util.regex.Matcher
import java.util.regex.Pattern

@Log
class RunPipelineCommand extends BpipeCommand {
@@ -44,13 +46,9 @@ class RunPipelineCommand extends BpipeCommand {

File dirFile = new File(dir).absoluteFile

// This was mainly for security, but is actually problematic because we need pipeline directories two deep at times
// TODO: figure out how to limit / constrain this - an entry in agent config?
//
// if(!dirFile.parentFile.exists())
// throw new IllegalArgumentException("Directory supplied $dir is not in an existing path. The directory parent must already exist.")
//
if(!dirFile.exists() && !dirFile.mkdirs())
createRunDirectory(dirFile)

if(!dirFile.exists())
throw new IllegalArgumentException("Unable to create directory requested for pipeline run $dir.")

log.info "Running with arguments: " + args;
@@ -61,5 +59,93 @@ class RunPipelineCommand extends BpipeCommand {
directory(dirFile)
}
}

/**
* This lock prevents multiple concurrent jobs launched by the bpipe
* agent from attempting to create the same directory when the directories
* are templated.
*/
private final static Object RUN_DIRECTORY_COMPUTE_LOCK = new Object()

/**
* Iterate the path hierarchy and create any directories that do not exist,
* examining each level for possible incrementing templates, where such a template
* is specified with <code>{inc}</code> in the body of a path.
*/
void createRunDirectory(File dirFile) {

synchronized(RUN_DIRECTORY_COMPUTE_LOCK) {

List<String> dirParts = dirFile.path.tokenize('/')
String parentPath = '/'
String resultPath = dirParts.collect { String part ->
computeRunDirectoryPart(parentPath, part)
parentPath = parentPath + '/' + part
}.join('/')

File resultFile = new File(resultPath)

if(resultFile.exists())
return

log.info "Creating directory path ${resultFile} to run command"

resultFile.mkdirs()
}
}

/**
* Regular expression to identify increment within paths
*/
public static final Pattern PATH_INCREMENTER_REGEX = ~/.*(\{inc\}).*/

/**
* Create the given childDir, replacing any incrementing portion
* of the path (specified by the form <code>{inc}</code> within the path.
*
* @return the created path
*/
String computeRunDirectoryPart(final File parentFile, final String childDir) {

assert parentFile.exists()
assert parentFile.isDirectory()

Matcher m = PATH_INCREMENTER_REGEX.matcher(childDir)
if(!m)
return childDir

List<Integer> matched_indices = findChildDirectoryIndices(parentFile,childDir)

log.info "Found ${matched_indices.size()} matching directories in dir ${parentFile}"

int nextValue = (matched_indices.max()?:0) + 1

return childDir.replaceFirst(/\{inc\}/, String.format('%04d', nextValue))
}

/**
* Identify directories within the given parent dir that match the
* form of the template given where {inc} is treated as a numeric
* wildcard.
*
* @param parentFile
* @param dirPart
* @return
*/
List<Integer> findChildDirectoryIndices(File parentFile, String template) {

Pattern incPattern = Pattern.compile(template.replaceFirst(/\{inc\}/, '([0-9]{1,6})'))

// Find all the dirs within the parent path that match the given expression
List<Integer> matching_dirs =
parentFile.listFiles()
.grep { File f -> f.isDirectory() }
.collect { incPattern.matcher(it.name) }
.grep { it.matches() }
.collect { it[0][1] } // first group from regex match match
.grep { it.isInteger() }
.collect { it.toInteger() }
return matching_dirs
}
}

@@ -0,0 +1,91 @@
package bpipe.cmd

import static org.junit.Assert.*

import org.junit.Test

class RunPipelineCommandTest {

File p

def rpc = new RunPipelineCommand(["pipeline.groovy"])

@Test
public void 'test finding templated folder names'() {
File p = testDir([
'foo_bar_001_baz',
'foo_bar_002_baz',
'foo_bar_007_baz',
])

List idxs = rpc.findChildDirectoryIndices(p, 'foo_bar_{inc}_baz')
assert idxs.size() == 3
assert idxs.max() == 7
}

@Test
public void 'test folder without numeric'() {
p = testDir([
'foo_bar_001_baz',
'foo_bar_002_baz',
'foo_bar_cat_baz',
'foo_bar_009_baz',
])

List idxs = rpc.findChildDirectoryIndices(p, 'foo_bar_{inc}_baz')
assert idxs.size() == 3
assert idxs.max() == 9
}

@Test
public void 'test mismatching prefix'() {
p = testDir([
'boo_bar_001_baz',
'foo_bar_002_baz',
'foo_bar_cat_baz',
'coo_bar_009_baz',
])

List idxs = rpc.findChildDirectoryIndices(p, 'foo_bar_{inc}_baz')
assert idxs.size() == 1
assert idxs.max() == 2
}

@Test
void 'test simple directory increment'() {
p = testDir([])

List idxs = [1,3,5]
def rpc = new RunPipelineCommand(["pipeline.groovy"]) {
List<Integer> findChildDirectoryIndices(File parentFile, String template) {
return idxs
}
}

String dir = rpc.computeRunDirectoryPart(p, 'cat_dog_{inc}_tree')
assert dir == 'cat_dog_0006_tree'

idxs = [9999]
dir = rpc.computeRunDirectoryPart(p, 'cat_dog_{inc}_tree')
assert dir == 'cat_dog_10000_tree'
}

@Test
public void 'test empty folder'() {
p = testDir([])
List idxs = rpc.findChildDirectoryIndices(p, 'foo_bar_{inc}_baz')
assert idxs.size() == 0
}

private File testDir(List<String> paths) {
File p = new File('test') {
boolean exists() { true }
boolean isDirectory() { true }
File[] listFiles() {
paths.collect { new File(it) { boolean isDirectory() { true }} }
}
}
return p
}

}

0 comments on commit d1a3aa5

Please sign in to comment.
You can’t perform that action at this time.