Fix Invalid GLS path composition in staging directories #1654

modules/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategy.groovy

@@ -79,9 +79,9 @@ class GoogleLifeSciencesFileCopyStrategy extends SimpleFileCopyStrategy {
             }
 
             if(storePathIsDir) {
-                stagingCommands << "$gsutilPrefix cp -R $escapedStoreUri/ $localTaskDir".toString()
+                stagingCommands << "$gsutilPrefix cp -R $escapedStoreUri $localTaskDir".toString()
                 //check if we need to move the directory (gsutil doesn't support renaming directories on copy)
-                if(parent || !storePath.toString().endsWith(stageName)) {
+                if(parent || !storePath.name.endsWith(stageName)) {
                     stagingCommands << "mv $localTaskDir/${Escape.path(storePath.name)} $localTaskDir/$escapedStageName".toString()
                 }
             } else {
@@ -137,6 +137,8 @@ class GoogleLifeSciencesFileCopyStrategy extends SimpleFileCopyStrategy {
     }
 
     String copyMany(String local, Path target) {
+        if ( local.endsWith("/") )
+            local = local.substring(0,local.length()-1)
         "IFS=\$'\\n'; for name in \$(eval \"ls -1d ${Escape.path(local)}\" 2>/dev/null);do gsutil -m -q cp -R \$name ${Escape.uriPath(target)}/\$name; done; unset IFS"
     }
 

modules/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategyTest.groovy

@@ -86,7 +86,7 @@ class GoogleLifeSciencesFileCopyStrategyTest extends GoogleSpecification {
         then:
         result == '''\
                 echo start | gsutil -q cp  -c - gs://my-bucket/work/xx/yy/.command.begin
-                gsutil -m -q cp -R gs://my-bucket/foo/dir1/ /work/xx/yy
+                gsutil -m -q cp -R gs://my-bucket/foo/dir1 /work/xx/yy
                 '''.stripIndent()
 
         // stage file is a directory with a different name
@@ -96,7 +96,7 @@ class GoogleLifeSciencesFileCopyStrategyTest extends GoogleSpecification {
         then:
         result == '''\
                 echo start | gsutil -q cp  -c - gs://my-bucket/work/xx/yy/.command.begin
-                gsutil -m -q cp -R gs://my-bucket/foo/dir1/ /work/xx/yy
+                gsutil -m -q cp -R gs://my-bucket/foo/dir1 /work/xx/yy
                 mv /work/xx/yy/dir1 /work/xx/yy/dir2
                 '''.stripIndent()
     }
@@ -131,7 +131,7 @@ class GoogleLifeSciencesFileCopyStrategyTest extends GoogleSpecification {
         then:
         result == '''\
                 echo start | gsutil -q cp  -c - gs://my-bucket/work/xx/yy/.command.begin
-                gsutil -m -q -u foo cp -R gs://my-bucket/foo/dir1/ /work/xx/yy
+                gsutil -m -q -u foo cp -R gs://my-bucket/foo/dir1 /work/xx/yy
                 '''.stripIndent()
 
     }
@@ -226,4 +226,30 @@ class GoogleLifeSciencesFileCopyStrategyTest extends GoogleSpecification {
             gsutil -m -q cp -P -r gs://my-bucket/bin/d\\ i\\ r/* /work/xx/yy/nextflow-bin
             '''.stripIndent()
     }
+
+
+    def 'should validate copy many' () {
+        given:
+        def handler = Mock(GoogleLifeSciencesTaskHandler) {
+            getExecutor() >> Mock(GoogleLifeSciencesExecutor) { getConfig() >> Mock(GoogleLifeSciencesConfig) }
+        }
+        def strategy = new GoogleLifeSciencesFileCopyStrategy(Mock(TaskBean), handler)
+
+        when:
+        def ret1 = strategy.copyMany('file.txt', mockGsPath('gs://foo/bar'))
+        then:
+        ret1 == 'IFS=$\'\\n\'; for name in $(eval "ls -1d file.txt" 2>/dev/null);do gsutil -m -q cp -R $name gs://foo/bar/$name; done; unset IFS'
+
+        when:
+        def ret2 = strategy.copyMany('file name', mockGsPath('gs://foo/bar'))
+        then:
+        ret2 == 'IFS=$\'\\n\'; for name in $(eval "ls -1d file\\ name" 2>/dev/null);do gsutil -m -q cp -R $name gs://foo/bar/$name; done; unset IFS'
+
+        when:
+        def ret3 = strategy.copyMany('dir-name/', mockGsPath('gs://foo/bar'))
+        then:
+        ret3 == 'IFS=$\'\\n\'; for name in $(eval "ls -1d dir-name" 2>/dev/null);do gsutil -m -q cp -R $name gs://foo/bar/$name; done; unset IFS'
+
+
+    }
 }

validation/awsbatch.config

@@ -1,6 +1,7 @@
 workDir = 's3://nextflow-ci/work'
 process.executor = 'awsbatch'
 process.queue = 'nextflow-ci'
+process.container = 'nextflow/rnaseq-nf:latest'
 aws.region = 'eu-west-1'
 aws.batch.cliPath = '/home/ec2-user/miniconda/bin/aws'
 aws.batch.maxTransferAttempts = 3 

validation/awsbatch.sh

@@ -34,8 +34,9 @@ $NXF_CMD run test-complexpaths.nf -resume -c awsbatch.config
 [[ -e 'foo/sample.zip' ]] || false
 [[ -e 'foo/sample_(1 2).vcf' ]] || false
 
+$NXF_CMD run test-subdirs.nf -c awsbatch.config
 
 $NXF_CMD run nextflow-io/rnaseq-nf \
     -profile batch \
     -with-report \
-    -with-trace
+    -with-trace

validation/gls.sh

@@ -59,3 +59,5 @@ $NXF_CMD -C ./gls.config run ./test-complexpaths.nf -resume
 [[ -e 'foo/sample.zip' ]] || false
 [[ -e 'foo/sample_(1 2).vcf' ]] || false
 
+## run test-subdirs inputs/outputs
+$NXF_CMD -C ./gls.config run ./test-subdirs.nf

validation/test-subdirs.nf

@@ -0,0 +1,53 @@
+process foo {
+    output:
+    path 'gsfolder/' into ch1
+    path 'gsfolder2' into ch2
+    path 'gsfolder3' into ch3
+    path 'gsfolder4' into ch4
+    path 'test5.txt' into ch5
+    path 'test6.txt' into ch6
+    path 'test7.txt' into ch7
+
+    """
+    mkdir -p gsfolder/sub
+    touch gsfolder/test1.txt
+    touch gsfolder/sub/test1.txt
+
+    mkdir gsfolder2
+    touch gsfolder2/test2.txt
+
+    mkdir gsfolder3
+    touch gsfolder3/test3.txt
+
+    mkdir gsfolder4
+    touch gsfolder4/test4.txt
+
+    echo 'Hello 5'>>test5.txt
+    echo 'Hello 6'>>test6.txt
+    echo 'Hello 7'>>test7.txt
+
+    """
+}
+
+process bar {
+    input:
+    path test_folder from ch1
+    path test_folder2 from ch2
+    path 'test-folder3' from ch3
+    path 'test-folder4/*' from ch4
+    path test5 from ch5
+    path 'this-is-test-6.txt' from ch6
+    path 'test7/foo/*' from ch7
+
+    """
+    set -x
+    [[ -f $test_folder/test1.txt ]] || false
+    [[ -f $test_folder/sub/test1.txt ]] || false
+    [[ -f $test_folder2/test2.txt ]] || false
+    [[ -f test-folder3/test3.txt ]] || false
+    [[ -f test-folder4/gsfolder4/test4.txt ]] || false
+    [[ \$(cat $test5) = 'Hello 5' ]] || false
+    [[ \$(cat this-is-test-6.txt) = 'Hello 6' ]] || false
+    [[ \$(cat test7/foo/test7.txt) = 'Hello 7' ]] || false
+    """
+}

validation/test.sh

@@ -58,13 +58,13 @@ $NXF_CMD run nextflow-io/rnaseq-nf -with-docker $OPTS -resume
 # AWS Batch tests
 #
 echo aws batch tests
-./await.sh bash awsbatch.sh
+bash awsbatch.sh
 
 #
 # Google Life Sciences
 #
 if [[ $GOOGLE_SECRET ]]; then
-  ./await.sh bash gls.sh
+  bash gls.sh
 else
   echo "Google Life Science test skipped because GOOGLE_SECRET env var is missing"
-fi
+fi