Improve control on azcopy install (#4883)

Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>
nextflow-io · Apr 5, 2024 · 01447d5 · 01447d5
1 parent a607f3f
commit 01447d5
Show file tree

Hide file tree

Showing 5 changed files with 39 additions and 20 deletions.
diff --git a/docs/config.md b/docs/config.md
@@ -341,7 +341,7 @@ The following settings are available:
 : Enable the automatic creation of batch pools depending on the pipeline resources demand (default: `true`).
 
 `azure.batch.copyToolInstallMode`
-: Specify where the `azcopy` tool used by Nextflow. When `node` is specified it's copied once during the pool creation. When `task` is provider, it's installed for each task execution (default: `node`).
+: Specify where the `azcopy` tool used by Nextflow. When `node` is specified it's copied once during the pool creation. When `task` is provider, it's installed for each task execution. Finally when `off` is specified, the `azcopy` tool is not installed (default: `node`).
 
 `azure.batch.deleteJobsOnCompletion`
 : Delete all jobs when the workflow completes (default: `false`).

diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy
@@ -681,17 +681,21 @@ class AzBatchService implements Closeable {
                 .withContainerConfiguration(containerConfig)
     }
 
-    protected void createPool(AzVmPoolSpec spec) {
-
-        def resourceFiles = new ArrayList(10)
+    protected StartTask createStartTask() {
+        if( config.batch().getCopyToolInstallMode() != CopyToolInstallMode.node )
+            return null
 
+        final resourceFiles = new ArrayList(10)
         resourceFiles << new ResourceFile()
-                .withHttpUrl(AZCOPY_URL)
-                .withFilePath('azcopy')
+            .withHttpUrl(AZCOPY_URL)
+            .withFilePath('azcopy')
+
+        return new StartTask()
+            .withCommandLine('bash -c "chmod +x azcopy && mkdir \$AZ_BATCH_NODE_SHARED_DIR/bin/ && cp azcopy \$AZ_BATCH_NODE_SHARED_DIR/bin/" ')
+            .withResourceFiles(resourceFiles)
+    }
 
-        def poolStartTask = new StartTask()
-                .withCommandLine('bash -c "chmod +x azcopy && mkdir \$AZ_BATCH_NODE_SHARED_DIR/bin/ && cp azcopy \$AZ_BATCH_NODE_SHARED_DIR/bin/" ')
-                .withResourceFiles(resourceFiles)
+    protected void createPool(AzVmPoolSpec spec) {
 
         final poolParams = new PoolAddParameter()
                 .withId(spec.poolId)
@@ -701,7 +705,11 @@ class AzBatchService implements Closeable {
                 // same as the num ofd cores
                 // https://docs.microsoft.com/en-us/azure/batch/batch-parallel-node-tasks
                 .withTaskSlotsPerNode(spec.vmType.numberOfCores)
-                .withStartTask(poolStartTask)
+
+        final startTask = createStartTask()
+        if( startTask ) {
+            poolParams .withStartTask(startTask)
+        }
 
         // resource labels
         if( spec.metadata ) {

diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzBatchOpts.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzBatchOpts.groovy
@@ -21,7 +21,10 @@ import java.util.regex.Matcher
 import java.util.regex.Pattern
 
 import groovy.transform.CompileStatic
+import nextflow.Global
+import nextflow.Session
 import nextflow.cloud.CloudTransferOptions
+import nextflow.fusion.FusionHelper
 import nextflow.util.Duration
 import nextflow.util.StringUtils
 
@@ -136,10 +139,12 @@ class AzBatchOpts implements CloudTransferOptions {
     CopyToolInstallMode getCopyToolInstallMode() {
         // if the `installAzCopy` is not specified
         // `true` is returned when the pool is not create by Nextflow
-        // since it can be a pol provided by the user which does not
+        // since it can be a pool provided by the user which does not
         // provide the required `azcopy` tool
         if( copyToolInstallMode )
             return copyToolInstallMode
+        if( FusionHelper.isFusionEnabled((Session) Global.session) )
+            return CopyToolInstallMode.off
         canCreatePool() ? CopyToolInstallMode.node : CopyToolInstallMode.task
     }
 }
diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/config/CopyToolInstallMode.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/config/CopyToolInstallMode.groovy
@@ -23,5 +23,6 @@ package nextflow.cloud.azure.config
  */
 enum CopyToolInstallMode {
     node,
-    task
+    task,
+    off
 }
diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzBatchOptsTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzBatchOptsTest.groovy
@@ -1,5 +1,7 @@
 package nextflow.cloud.azure.config
 
+import nextflow.Global
+import nextflow.Session
 import spock.lang.Specification
 import spock.lang.Unroll
 
@@ -67,20 +69,23 @@ class AzBatchOptsTest extends Specification {
     @Unroll
     def 'should check azcopy install' () {
         given:
+        Global.session = Mock(Session) { getConfig()>>[fusion:[enabled: FUSION]] }
         AzBatchOpts opts = Spy(AzBatchOpts, constructorArgs: [ CONFIG,[:] ])
 
         expect:
         opts.getCopyToolInstallMode() == EXPECTED
 
         where:
-        EXPECTED                    | CONFIG
-        CopyToolInstallMode.task    | [:]
-        CopyToolInstallMode.node    | [allowPoolCreation: true]
-        CopyToolInstallMode.node    | [autoPoolMode: true]
-        CopyToolInstallMode.node    | [allowPoolCreation: true, copyToolInstallMode: 'node']
-        CopyToolInstallMode.task    | [allowPoolCreation: true, copyToolInstallMode: 'task']
-        CopyToolInstallMode.task    | [copyToolInstallMode: 'task']
-        CopyToolInstallMode.node    | [copyToolInstallMode: 'node']
+        EXPECTED                    | CONFIG                                                    | FUSION
+        CopyToolInstallMode.task    | [:]                                                       | false
+        CopyToolInstallMode.node    | [allowPoolCreation: true]                                 | false
+        CopyToolInstallMode.node    | [autoPoolMode: true]                                      | false
+        CopyToolInstallMode.node    | [allowPoolCreation: true, copyToolInstallMode: 'node']    | false
+        CopyToolInstallMode.task    | [allowPoolCreation: true, copyToolInstallMode: 'task']    | false
+        CopyToolInstallMode.task    | [copyToolInstallMode: 'task']                             | false
+        CopyToolInstallMode.node    | [copyToolInstallMode: 'node']                             | false
+        CopyToolInstallMode.off     | [copyToolInstallMode: 'off']                              | false
+        CopyToolInstallMode.off     | [:]                                                       | true
 
     }
 }