Use account SAS token instead of service SAS token (#2576)

Signed-off-by: fbdtemme <florian.detemmerman@lizard.bio> Co-authored-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>
nextflow-io · Jan 20, 2022 · d512597 · d512597
1 parent 5ba16dd
commit d512597
Show file tree

Hide file tree

Showing 4 changed files with 68 additions and 3 deletions.
diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchExecutor.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchExecutor.groovy
@@ -94,9 +94,9 @@ class AzBatchExecutor extends Executor implements ExtensionPoint {
     protected void initBatchService() {
         config = AzConfig.getConfig(session)
         batchService = new AzBatchService(this)
-        // generate a SAS token if missing
+        // generate an account SAS token if missing
         if( !config.storage().sasToken )
-            config.storage().sasToken = AzHelper.generateContainerSas(workDir, config.storage().tokenDuration)
+            config.storage().sasToken = AzHelper.generateAccountSas(workDir, config.storage().tokenDuration)
 
         session.onShutdown { batchService.close() }
     }

diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzHelper.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzHelper.groovy
@@ -14,6 +14,11 @@
  * limitations under the License.
  */
 package nextflow.cloud.azure.batch
+import com.azure.storage.blob.BlobServiceClient
+import com.azure.storage.common.sas.AccountSasPermission
+import com.azure.storage.common.sas.AccountSasResourceType
+import com.azure.storage.common.sas.AccountSasService
+import com.azure.storage.common.sas.AccountSasSignatureValues
 
 import java.nio.file.Path
 import java.time.OffsetDateTime
@@ -55,6 +60,10 @@ class AzHelper {
         generateSas(az0(path).containerClient(), duration)
     }
 
+    static String generateAccountSas(Path path, Duration duration) {
+        generateAccountSas(az0(path).getFileSystem().getBlobServiceClient(), duration)
+    }
+
     static BlobContainerSasPermission CONTAINER_PERMS = new BlobContainerSasPermission()
             .setAddPermission(true)
             .setCreatePermission(true)
@@ -75,6 +84,24 @@ class AzHelper {
             .setTagsPermission(true)
             .setWritePermission(true)
 
+    static AccountSasPermission ACCOUNT_PERMS = new AccountSasPermission()
+            .setAddPermission(true)
+            .setCreatePermission(true)
+            .setDeletePermission(true)
+            .setListPermission(true)
+            .setReadPermission(true)
+            .setTagsPermission(true)
+            .setWritePermission(true)
+            .setUpdatePermission(true)
+
+    static AccountSasService ACCOUNT_SERVICES = new AccountSasService()
+            .setBlobAccess(true)
+            .setFileAccess(true)
+
+    static AccountSasResourceType ACCOUNT_RESOURCES = new AccountSasResourceType()
+            .setContainer(true)
+            .setObject(true)
+            .setService(true)
 
     static String generateSas(BlobContainerClient client, Duration duration) {
         final now = OffsetDateTime .now()
@@ -88,4 +115,14 @@ class AzHelper {
         return client .generateSas(signature)
     }
 
+    static String generateAccountSas(BlobServiceClient client, Duration duration) {
+        final expiryTime = OffsetDateTime.now().plusSeconds(duration.toSeconds());
+        final signature = new AccountSasSignatureValues(
+                expiryTime,
+                ACCOUNT_PERMS,
+                ACCOUNT_SERVICES,
+                ACCOUNT_RESOURCES)
+
+        return client.generateAccountSas(signature)
+    }
 }
diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileSystem.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileSystem.groovy
@@ -408,8 +408,19 @@ class AzFileSystem extends FileSystem {
 
     @PackageScope
     void copy(AzPath source, AzPath target) {
+        final sasToken = provider.getSasToken()
+        String sourceUrl = source.blobClient().getBlobUrl()
+
+        if (sasToken != null) {
+            if (sourceUrl.contains('?')){
+                sourceUrl = String.format("%s&%s", sourceUrl, sasToken);
+            } else {
+                sourceUrl = String.format("%s?%s", sourceUrl, sasToken);
+            }
+        }
+
         SyncPoller<BlobCopyInfo, Void> pollResponse =
-                target.blobClient().beginCopy( source.blobClient().getBlobUrl(), null )
+                target.blobClient().beginCopy( sourceUrl, null )
         pollResponse.waitForCompletion(Duration.ofSeconds(maxCopyDurationSecs))
     }
 

diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileSystemProvider.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileSystemProvider.groovy
@@ -62,6 +62,8 @@ class AzFileSystemProvider extends FileSystemProvider {
 
     private Map<String,String> env = new HashMap<>(System.getenv())
     private Map<String,AzFileSystem> fileSystems = [:]
+    private String sasToken = null
+    private String accountKey = null
 
     /**
      * @inheritDoc
@@ -71,6 +73,14 @@ class AzFileSystemProvider extends FileSystemProvider {
         return SCHEME
     }
 
+    String getSasToken() {
+        return this.sasToken
+    }
+
+    String getAccountKey() {
+        return this.accountKey
+    }
+
     static private AzPath asAzPath(Path path ) {
         if( path !instanceof AzPath )
             throw new IllegalArgumentException("Not a valid Azure blob storage path object: `$path` [${path?.class?.name?:'-'}]" )
@@ -197,6 +207,13 @@ class AzFileSystemProvider extends FileSystemProvider {
                 : createBlobServiceWithKey(accountName, accountKey)
         final result = createFileSystem(client, bucket, config)
         fileSystems[bucket] = result
+
+        if (sasToken) {
+            this.sasToken = sasToken
+        }
+        if (accountKey) {
+            this.accountKey = accountKey
+        }
         return result
     }