Skip to content

Commit

Permalink
Add readOnlyInputs option for CharliecloudBuilder (#3590)
Browse files Browse the repository at this point in the history 
Signed-off-by: Laurent Modolo <laurent.modolo@ens-lyon.fr>
  • Loading branch information
@l-modolo
l-modolo committed Feb 2, 2023
1 parent c0d4aab commit 8ae001f
Show file tree
Hide file tree
Showing 3 changed files with 67 additions and 13 deletions.
26 changes: 23 additions & 3 deletions modules/nextflow/src/main/groovy/nextflow/container/CharliecloudBuilder.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import groovy.util.logging.Slf4j
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
* @author Patrick Hüther <patrick.huether@gmail.com>
* @author Laurent Modolo <laurent.modolo@ens-lyon.fr>
*/
@CompileStatic
@Slf4j
Expand All @@ -46,6 +47,9 @@ class CharliecloudBuilder extends ContainerBuilder<CharliecloudBuilder> {
if( params.containsKey('runOptions') )
addRunOptions(params.runOptions.toString())

if( params.containsKey('readOnlyInputs') )
this.readOnlyInputs = params.readOnlyInputs?.toString() == 'true'

return this
}

Expand All @@ -58,7 +62,9 @@ class CharliecloudBuilder extends ContainerBuilder<CharliecloudBuilder> {
CharliecloudBuilder build(StringBuilder result) {
assert image

result << 'ch-run --unset-env="*" -c "$PWD" -w --no-home --set-env '
result << 'ch-run --unset-env="*" -c "$PWD" --no-home --set-env '
if (!readOnlyInputs)
result << '-w '

appendEnv(result)

Expand All @@ -78,9 +84,23 @@ class CharliecloudBuilder extends ContainerBuilder<CharliecloudBuilder> {
return this
}

protected String getRoot(String path) {
def rootPath = path.tokenize("/")

if (rootPath.size() >= 1 && path[0] == '/')
rootPath = "/${rootPath[0]}"
else
throw new IllegalArgumentException("Not a valid working directory value (absolute path?): ${path}")

return rootPath
}

@Override
protected String composeVolumePath(String path, boolean readOnly = false) {
return "-b ${escape(path)}"
protected String composeVolumePath(String path, boolean readOnlyInputs = false) {
def mountCmd = "-b ${escape(path)}"
if (readOnlyInputs)
mountCmd = "-b ${getRoot(escape(path))}"
return mountCmd
}

@Override
Expand Down
3 changes: 2 additions & 1 deletion modules/nextflow/src/main/groovy/nextflow/container/CharliecloudCache.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,8 @@ class CharliecloudCache {
return localPath
}

final file = new File("${localPath.parent.parent.parent}/.${localPath.name}.lock")
// final file = new File("${localPath.parent.parent.parent}/.${localPath.name}.lock")
final file = new File("${localPath.parent.parent.parent}/.ch-pulling.lock")
final wait = "Another Nextflow instance is pulling the image $imageUrl with Charliecloud -- please wait until the download completes"
final err = "Unable to acquire exclusive lock after $pullTimeout on file: $file"

Expand Down
51 changes: 42 additions & 9 deletions modules/nextflow/src/test/groovy/nextflow/container/CharliecloudBuilderTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import spock.lang.Unroll
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
* @author Patrick Hüther <patrick.huether@gmail.com>
* @author Laurent Modolo <laurent.modolo@ens-lyon.fr>
*/
class CharliecloudBuilderTest extends Specification {

Expand All @@ -38,52 +39,84 @@ class CharliecloudBuilderTest extends Specification {
expect:
new CharliecloudBuilder('busybox')
.build()
.runCommand == 'ch-run --unset-env="*" -c "$PWD" -w --no-home --set-env -b "$PWD" busybox --'
.runCommand == 'ch-run --unset-env="*" -c "$PWD" --no-home --set-env -w -b "$PWD" busybox --'

new CharliecloudBuilder('busybox')
.params(runOptions: '-j')
.build()
.runCommand == 'ch-run --unset-env="*" -c "$PWD" -w --no-home --set-env -b "$PWD" -j busybox --'
.runCommand == 'ch-run --unset-env="*" -c "$PWD" --no-home --set-env -w -b "$PWD" -j busybox --'

new CharliecloudBuilder('busybox')
.params(temp: '/foo')
.build()
.runCommand == 'ch-run --unset-env="*" -c "$PWD" -w --no-home --set-env -b /foo:/tmp -b "$PWD" busybox --'
.runCommand == 'ch-run --unset-env="*" -c "$PWD" --no-home --set-env -w -b /foo:/tmp -b "$PWD" busybox --'

new CharliecloudBuilder('busybox')
.addEnv('X=1')
.addEnv(ALPHA:'aaa', BETA: 'bbb')
.build()
.runCommand == 'ch-run --unset-env="*" -c "$PWD" -w --no-home --set-env --set-env=X=1 --set-env=ALPHA=aaa --set-env=BETA=bbb -b "$PWD" busybox --'
.runCommand == 'ch-run --unset-env="*" -c "$PWD" --no-home --set-env -w --set-env=X=1 --set-env=ALPHA=aaa --set-env=BETA=bbb -b "$PWD" busybox --'

new CharliecloudBuilder('ubuntu')
.addMount(path1)
.build()
.runCommand == 'ch-run --unset-env="*" -c "$PWD" -w --no-home --set-env -b /foo/data/file1 -b "$PWD" ubuntu --'
.runCommand == 'ch-run --unset-env="*" -c "$PWD" --no-home --set-env -w -b /foo/data/file1 -b "$PWD" ubuntu --'

new CharliecloudBuilder('ubuntu')
.addMount(path1)
.addMount(path2)
.build()
.runCommand == 'ch-run --unset-env="*" -c "$PWD" -w --no-home --set-env -b /foo/data/file1 -b /bar/data/file2 -b "$PWD" ubuntu --'
.runCommand == 'ch-run --unset-env="*" -c "$PWD" --no-home --set-env -w -b /foo/data/file1 -b /bar/data/file2 -b "$PWD" ubuntu --'
}

def db_file = Paths.get('/home/db')
def 'should get run command' () {

when:
def cmd = new CharliecloudBuilder('ubuntu').build().getRunCommand()
then:
cmd == 'ch-run --unset-env="*" -c "$PWD" -w --no-home --set-env -b "$PWD" ubuntu --'
cmd == 'ch-run --unset-env="*" -c "$PWD" --no-home --set-env -w -b "$PWD" ubuntu --'

when:
cmd = new CharliecloudBuilder('ubuntu').build().getRunCommand('bwa --this --that file.fastq')
then:
cmd == 'ch-run --unset-env="*" -c "$PWD" -w --no-home --set-env -b "$PWD" ubuntu -- bwa --this --that file.fastq'
cmd == 'ch-run --unset-env="*" -c "$PWD" --no-home --set-env -w -b "$PWD" ubuntu -- bwa --this --that file.fastq'

when:
cmd = new CharliecloudBuilder('ubuntu').params(entry:'/bin/sh').build().getRunCommand('bwa --this --that file.fastq')
then:
cmd == 'ch-run --unset-env="*" -c "$PWD" -w --no-home --set-env -b "$PWD" ubuntu -- /bin/sh -c "bwa --this --that file.fastq"'
cmd == 'ch-run --unset-env="*" -c "$PWD" --no-home --set-env -w -b "$PWD" ubuntu -- /bin/sh -c "bwa --this --that file.fastq"'

when:
cmd = new CharliecloudBuilder('ubuntu').params(entry:'/bin/sh').params(readOnlyInputs: 'true').build().getRunCommand('bwa --this --that file.fastq')
then:
cmd == 'ch-run --unset-env="*" -c "$PWD" --no-home --set-env -b "$PWD" ubuntu -- /bin/sh -c "bwa --this --that file.fastq"'

when:
cmd = new CharliecloudBuilder('ubuntu').params(entry:'/bin/sh').params(readOnlyInputs: 'false').build().getRunCommand('bwa --this --that file.fastq')
then:
cmd == 'ch-run --unset-env="*" -c "$PWD" --no-home --set-env -w -b "$PWD" ubuntu -- /bin/sh -c "bwa --this --that file.fastq"'

when:
cmd = new CharliecloudBuilder('ubuntu')
.params(entry:'/bin/sh')
.addMount(db_file)
.addMount(db_file)
.params(readOnlyInputs: 'true')
.build().getRunCommand('bwa --this --that file.fastq')
then:
cmd == 'ch-run --unset-env="*" -c "$PWD" --no-home --set-env -b /home -b "$PWD" ubuntu -- /bin/sh -c "bwa --this --that file.fastq"'

when:
cmd = new CharliecloudBuilder('ubuntu')
.params(entry:'/bin/sh')
.addMount(db_file)
.addMount(db_file)
.params(readOnlyInputs: 'false')
.build()
.getRunCommand('bwa --this --that file.fastq')
then:
cmd == 'ch-run --unset-env="*" -c "$PWD" --no-home --set-env -w -b /home/db -b "$PWD" ubuntu -- /bin/sh -c "bwa --this --that file.fastq"'
}

@Unroll
Expand Down

0 comments on commit 8ae001f

Please sign in to comment.