diff --git a/docs/config.md b/docs/config.md index c0410435b1..123d032f8e 100644 --- a/docs/config.md +++ b/docs/config.md @@ -341,7 +341,7 @@ The following settings are available: : Enable the automatic creation of batch pools depending on the pipeline resources demand (default: `true`). `azure.batch.copyToolInstallMode` -: Specify where the `azcopy` tool used by Nextflow. When `node` is specified it's copied once during the pool creation. When `task` is provider, it's installed for each task execution (default: `node`). +: Specify where the `azcopy` tool used by Nextflow. When `node` is specified it's copied once during the pool creation. When `task` is provider, it's installed for each task execution. Finally when `off` is specified, the `azcopy` tool is not installed (default: `node`). `azure.batch.deleteJobsOnCompletion` : Delete all jobs when the workflow completes (default: `false`). diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy index 4cc6523f53..938bdb423b 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy @@ -681,17 +681,21 @@ class AzBatchService implements Closeable { .withContainerConfiguration(containerConfig) } - protected void createPool(AzVmPoolSpec spec) { - - def resourceFiles = new ArrayList(10) + protected StartTask createStartTask() { + if( config.batch().getCopyToolInstallMode() != CopyToolInstallMode.node ) + return null + final resourceFiles = new ArrayList(10) resourceFiles << new ResourceFile() - .withHttpUrl(AZCOPY_URL) - .withFilePath('azcopy') + .withHttpUrl(AZCOPY_URL) + .withFilePath('azcopy') + + return new StartTask() + .withCommandLine('bash -c "chmod +x azcopy && mkdir \$AZ_BATCH_NODE_SHARED_DIR/bin/ && cp azcopy \$AZ_BATCH_NODE_SHARED_DIR/bin/" ') + .withResourceFiles(resourceFiles) + } - def poolStartTask = new StartTask() - .withCommandLine('bash -c "chmod +x azcopy && mkdir \$AZ_BATCH_NODE_SHARED_DIR/bin/ && cp azcopy \$AZ_BATCH_NODE_SHARED_DIR/bin/" ') - .withResourceFiles(resourceFiles) + protected void createPool(AzVmPoolSpec spec) { final poolParams = new PoolAddParameter() .withId(spec.poolId) @@ -701,7 +705,11 @@ class AzBatchService implements Closeable { // same as the num ofd cores // https://docs.microsoft.com/en-us/azure/batch/batch-parallel-node-tasks .withTaskSlotsPerNode(spec.vmType.numberOfCores) - .withStartTask(poolStartTask) + + final startTask = createStartTask() + if( startTask ) { + poolParams .withStartTask(startTask) + } // resource labels if( spec.metadata ) { diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzBatchOpts.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzBatchOpts.groovy index 7bafefbfe7..358c70413d 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzBatchOpts.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzBatchOpts.groovy @@ -21,7 +21,10 @@ import java.util.regex.Matcher import java.util.regex.Pattern import groovy.transform.CompileStatic +import nextflow.Global +import nextflow.Session import nextflow.cloud.CloudTransferOptions +import nextflow.fusion.FusionHelper import nextflow.util.Duration import nextflow.util.StringUtils @@ -136,10 +139,12 @@ class AzBatchOpts implements CloudTransferOptions { CopyToolInstallMode getCopyToolInstallMode() { // if the `installAzCopy` is not specified // `true` is returned when the pool is not create by Nextflow - // since it can be a pol provided by the user which does not + // since it can be a pool provided by the user which does not // provide the required `azcopy` tool if( copyToolInstallMode ) return copyToolInstallMode + if( FusionHelper.isFusionEnabled((Session) Global.session) ) + return CopyToolInstallMode.off canCreatePool() ? CopyToolInstallMode.node : CopyToolInstallMode.task } } diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/config/CopyToolInstallMode.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/config/CopyToolInstallMode.groovy index 67b977e57f..981398fb07 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/config/CopyToolInstallMode.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/config/CopyToolInstallMode.groovy @@ -23,5 +23,6 @@ package nextflow.cloud.azure.config */ enum CopyToolInstallMode { node, - task + task, + off } diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzBatchOptsTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzBatchOptsTest.groovy index bba04f3aaa..8c6e0159e3 100644 --- a/plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzBatchOptsTest.groovy +++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzBatchOptsTest.groovy @@ -1,5 +1,7 @@ package nextflow.cloud.azure.config +import nextflow.Global +import nextflow.Session import spock.lang.Specification import spock.lang.Unroll @@ -67,20 +69,23 @@ class AzBatchOptsTest extends Specification { @Unroll def 'should check azcopy install' () { given: + Global.session = Mock(Session) { getConfig()>>[fusion:[enabled: FUSION]] } AzBatchOpts opts = Spy(AzBatchOpts, constructorArgs: [ CONFIG,[:] ]) expect: opts.getCopyToolInstallMode() == EXPECTED where: - EXPECTED | CONFIG - CopyToolInstallMode.task | [:] - CopyToolInstallMode.node | [allowPoolCreation: true] - CopyToolInstallMode.node | [autoPoolMode: true] - CopyToolInstallMode.node | [allowPoolCreation: true, copyToolInstallMode: 'node'] - CopyToolInstallMode.task | [allowPoolCreation: true, copyToolInstallMode: 'task'] - CopyToolInstallMode.task | [copyToolInstallMode: 'task'] - CopyToolInstallMode.node | [copyToolInstallMode: 'node'] + EXPECTED | CONFIG | FUSION + CopyToolInstallMode.task | [:] | false + CopyToolInstallMode.node | [allowPoolCreation: true] | false + CopyToolInstallMode.node | [autoPoolMode: true] | false + CopyToolInstallMode.node | [allowPoolCreation: true, copyToolInstallMode: 'node'] | false + CopyToolInstallMode.task | [allowPoolCreation: true, copyToolInstallMode: 'task'] | false + CopyToolInstallMode.task | [copyToolInstallMode: 'task'] | false + CopyToolInstallMode.node | [copyToolInstallMode: 'node'] | false + CopyToolInstallMode.off | [copyToolInstallMode: 'off'] | false + CopyToolInstallMode.off | [:] | true } }