Encryption v2 perf updates (#29768)

* Added encryption option to perf tests

* ci fixes and pr feedback

* Match encryption parameter with Python

* Fixing some perf test stuff

* Fixed perf tests

* Added abstract upload and download tests

* Added changes to perf configs

* Fixed bug about trying to get block blob clients from encrypted clients

* Fixed another bug

* Fix greedy operators

* Reverted ref to tools draft pr. Pr feedback

eng/pipelines/templates/jobs/perf.yml

@@ -37,7 +37,6 @@ resources:
     type: github
     endpoint: Azure
     name: Azure/azure-sdk-tools
-    ref: main
 
 jobs:
 - job: Perf

sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/DecryptorV2.java

@@ -52,7 +52,7 @@ Flux<ByteBuffer> decrypt(Flux<ByteBuffer> encryptedFlux, EncryptedBlobRange encr
                 gcmEncryptionRegionLength + TAG_LENGTH + nonceLength);
 
         return encryptedFlux
-            .flatMapSequential(stagingArea::write)
+            .flatMapSequential(stagingArea::write, 1, 1)
             .concatWith(Flux.defer(stagingArea::flush))
             .flatMapSequential(aggregator -> {
                 // Get the IV out of the beginning of the aggregator

sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/EncryptorV2.java

@@ -84,7 +84,7 @@ protected Flux<ByteBuffer> encrypt(Flux<ByteBuffer> plainTextFlux) {
             UploadUtils.chunkSource(plainTextFlux,
                     new com.azure.storage.common.ParallelTransferOptions()
                         .setBlockSizeLong((long) GCM_ENCRYPTION_REGION_LENGTH))
-                .flatMapSequential(stagingArea::write)
+                .flatMapSequential(stagingArea::write, 1, 1)
                 .concatWith(Flux.defer(stagingArea::flush))
                 .index()
                 .flatMapSequential(tuple -> {

sdk/storage/azure-storage-perf/src/main/java/com/azure/storage/blob/perf/BlobPerfStressOptions.java

@@ -4,15 +4,14 @@
 package com.azure.storage.blob.perf;
 
 import com.azure.storage.StoragePerfStressOptions;
-import com.azure.storage.blob.specialized.cryptography.EncryptionVersion;
 import com.beust.jcommander.Parameter;
 
 public class BlobPerfStressOptions extends StoragePerfStressOptions {
 
-    @Parameter(names = { "--encryption-version" })
-    private EncryptionVersion encryptionVersion = null;
+    @Parameter(names = { "--client-encryption" })
+    private String clientEncryption = null;
 
-    public EncryptionVersion getEncryptionVersion() {
-        return encryptionVersion;
+    public String getClientEncryption() {
+        return clientEncryption;
     }
 }

sdk/storage/azure-storage-perf/src/main/java/com/azure/storage/blob/perf/DownloadBlobNonSharedClientTest.java

@@ -4,19 +4,15 @@
 package com.azure.storage.blob.perf;
 
 import com.azure.perf.test.core.NullOutputStream;
-import com.azure.perf.test.core.PerfStressOptions;
 import com.azure.storage.blob.BlobAsyncClient;
 import com.azure.storage.blob.BlobClient;
 import com.azure.storage.blob.BlobClientBuilder;
-import com.azure.storage.blob.perf.core.BlobTestBase;
-import com.azure.storage.blob.perf.core.ContainerTest;
+import com.azure.storage.blob.perf.core.AbstractDownloadTest;
 import reactor.core.publisher.Mono;
 
 import java.io.OutputStream;
 
-import static com.azure.perf.test.core.TestDataCreationHelper.createRandomByteBufferFlux;
-
-public class DownloadBlobNonSharedClientTest extends BlobTestBase<BlobPerfStressOptions> {
+public class DownloadBlobNonSharedClientTest extends AbstractDownloadTest<BlobPerfStressOptions> {
     private static final int BUFFER_SIZE = 16 * 1024 * 1024;
     private static final OutputStream DEV_NULL = new NullOutputStream();
     String blobName = "downloadTest";
@@ -27,13 +23,6 @@ public DownloadBlobNonSharedClientTest(BlobPerfStressOptions options) {
         super(options);
     }
 
-    // Required resource setup goes here, upload the file to be downloaded during tests.
-    public Mono<Void> globalSetupAsync() {
-        return super.globalSetupAsync()
-                   .then(blobAsyncClient.upload(createRandomByteBufferFlux(options.getSize()), null))
-                   .then();
-    }
-
     // Perform the API call to be tested here
     @Override
     public void run() {

sdk/storage/azure-storage-perf/src/main/java/com/azure/storage/blob/perf/DownloadBlobTest.java

@@ -4,18 +4,12 @@
 package com.azure.storage.blob.perf;
 
 import com.azure.perf.test.core.NullOutputStream;
-import com.azure.perf.test.core.PerfStressOptions;
-import com.azure.storage.blob.BlobAsyncClient;
-import com.azure.storage.blob.BlobClient;
-import com.azure.storage.blob.perf.core.BlobTestBase;
-import com.azure.storage.blob.perf.core.ContainerTest;
+import com.azure.storage.blob.perf.core.AbstractDownloadTest;
 import reactor.core.publisher.Mono;
 
 import java.io.OutputStream;
 
-import static com.azure.perf.test.core.TestDataCreationHelper.createRandomByteBufferFlux;
-
-public class DownloadBlobTest extends BlobTestBase<BlobPerfStressOptions> {
+public class DownloadBlobTest extends AbstractDownloadTest<BlobPerfStressOptions> {
     private static final int BUFFER_SIZE = 16 * 1024 * 1024;
     private static final OutputStream DEV_NULL = new NullOutputStream();
 
@@ -25,12 +19,6 @@ public DownloadBlobTest(BlobPerfStressOptions options) {
         super(options);
     }
 
-    // Required resource setup goes here, upload the file to be downloaded during tests.
-    public Mono<Void> globalSetupAsync() {
-        return super.globalSetupAsync()
-            .then(blobAsyncClient.upload(createRandomByteBufferFlux(options.getSize()), null))
-            .then();
-    }
 
     // Perform the API call to be tested here
     @Override

sdk/storage/azure-storage-perf/src/main/java/com/azure/storage/blob/perf/DownloadBlobToFileTest.java

@@ -3,10 +3,7 @@
 
 package com.azure.storage.blob.perf;
 
-import com.azure.perf.test.core.PerfStressOptions;
-import com.azure.storage.blob.BlobAsyncClient;
-import com.azure.storage.blob.BlobClient;
-import com.azure.storage.blob.perf.core.BlobTestBase;
+import com.azure.storage.blob.perf.core.AbstractDownloadTest;
 import reactor.core.publisher.Mono;
 
 import java.io.File;
@@ -15,16 +12,14 @@
 import java.nio.file.Files;
 import java.util.UUID;
 
-import static com.azure.perf.test.core.TestDataCreationHelper.createRandomByteBufferFlux;
-
 /**
  * Use {@code downloadtofiletest} command to run this test.
  * This test requires providing connection string in {@code STORAGE_CONNECTION_STRING} environment variable.
  * It's recommended to use premium blob storage account.
  * This test includes temporary file deletion as part of scenario. Please keep in mind that this adds
  * constant component to the results.
  */
-public class DownloadBlobToFileTest extends BlobTestBase<BlobPerfStressOptions> {
+public class DownloadBlobToFileTest extends AbstractDownloadTest<BlobPerfStressOptions> {
 
     private final File tempDir;
 
@@ -39,12 +34,6 @@ public DownloadBlobToFileTest(BlobPerfStressOptions options) {
         }
     }
 
-    public Mono<Void> globalSetupAsync() {
-        return super.globalSetupAsync()
-            .then(blobAsyncClient.upload(createRandomByteBufferFlux(options.getSize()), null))
-            .then();
-    }
-
     @Override
     public void run() {
         File file = new File(tempDir, UUID.randomUUID().toString());

sdk/storage/azure-storage-perf/src/main/java/com/azure/storage/blob/perf/UploadBlobNoLengthTest.java

@@ -4,10 +4,9 @@
 package com.azure.storage.blob.perf;
 
 import com.azure.perf.test.core.RepeatingInputStream;
-import com.azure.storage.StoragePerfStressOptions;
 import com.azure.storage.blob.models.ParallelTransferOptions;
 import com.azure.storage.blob.options.BlobParallelUploadOptions;
-import com.azure.storage.blob.perf.core.BlobTestBase;
+import com.azure.storage.blob.perf.core.AbstractUploadTest;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 
@@ -16,7 +15,7 @@
 import static com.azure.perf.test.core.TestDataCreationHelper.createRandomByteBufferFlux;
 import static com.azure.perf.test.core.TestDataCreationHelper.createRandomInputStream;
 
-public class UploadBlobNoLengthTest extends BlobTestBase<BlobPerfStressOptions> {
+public class UploadBlobNoLengthTest extends AbstractUploadTest<BlobPerfStressOptions> {
     protected final RepeatingInputStream inputStream;
     protected final Flux<ByteBuffer> byteBufferFlux;
 

sdk/storage/azure-storage-perf/src/main/java/com/azure/storage/blob/perf/UploadBlobTest.java

@@ -4,25 +4,18 @@
 package com.azure.storage.blob.perf;
 
 import com.azure.perf.test.core.RepeatingInputStream;
-import com.azure.security.keyvault.keys.cryptography.models.KeyWrapAlgorithm;
-import com.azure.storage.StoragePerfStressOptions;
-import com.azure.storage.blob.BlobAsyncClient;
-import com.azure.storage.blob.BlobClient;
 import com.azure.storage.blob.models.ParallelTransferOptions;
 import com.azure.storage.blob.options.BlobParallelUploadOptions;
-import com.azure.storage.blob.perf.core.BlobTestBase;
-import com.azure.storage.blob.specialized.cryptography.EncryptedBlobClientBuilder;
-import com.azure.storage.blob.specialized.cryptography.EncryptionVersion;
+import com.azure.storage.blob.perf.core.AbstractUploadTest;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 
 import java.nio.ByteBuffer;
-import java.util.Random;
 
 import static com.azure.perf.test.core.TestDataCreationHelper.createRandomByteBufferFlux;
 import static com.azure.perf.test.core.TestDataCreationHelper.createRandomInputStream;
 
-public class UploadBlobTest extends BlobTestBase<BlobPerfStressOptions> {
+public class UploadBlobTest extends AbstractUploadTest<BlobPerfStressOptions> {
     protected final RepeatingInputStream inputStream;
     protected final Flux<ByteBuffer> byteBufferFlux;
 

sdk/storage/azure-storage-perf/src/main/java/com/azure/storage/blob/perf/UploadBlockBlobTest.java

@@ -3,9 +3,8 @@
 
 package com.azure.storage.blob.perf;
 
-import com.azure.perf.test.core.PerfStressOptions;
 import com.azure.perf.test.core.RepeatingInputStream;
-import com.azure.storage.blob.perf.core.BlobTestBase;
+import com.azure.storage.blob.perf.core.AbstractUploadTest;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 
@@ -14,7 +13,7 @@
 import static com.azure.perf.test.core.TestDataCreationHelper.createRandomByteBufferFlux;
 import static com.azure.perf.test.core.TestDataCreationHelper.createRandomInputStream;
 
-public class UploadBlockBlobTest extends BlobTestBase<BlobPerfStressOptions> {
+public class UploadBlockBlobTest extends AbstractUploadTest<BlobPerfStressOptions> {
     protected final RepeatingInputStream inputStream;
     protected final Flux<ByteBuffer> byteBufferFlux;
 

sdk/storage/azure-storage-perf/src/main/java/com/azure/storage/blob/perf/UploadFromFileTest.java

@@ -3,16 +3,15 @@
 
 package com.azure.storage.blob.perf;
 
-import com.azure.perf.test.core.PerfStressOptions;
 import com.azure.perf.test.core.TestDataCreationHelper;
-import com.azure.storage.blob.perf.core.BlobTestBase;
+import com.azure.storage.blob.perf.core.AbstractUploadTest;
 import reactor.core.publisher.Mono;
 
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
-public class UploadFromFileTest extends BlobTestBase<BlobPerfStressOptions> {
+public class UploadFromFileTest extends AbstractUploadTest<BlobPerfStressOptions> {
 
     private static final Path TEMP_FILE;
     private static final String TEMP_FILE_PATH;

sdk/storage/azure-storage-perf/src/main/java/com/azure/storage/blob/perf/UploadOutputStreamTest.java

@@ -3,18 +3,17 @@
 
 package com.azure.storage.blob.perf;
 
-import com.azure.storage.StoragePerfStressOptions;
 import com.azure.storage.blob.models.ParallelTransferOptions;
 import com.azure.storage.blob.options.BlockBlobOutputStreamOptions;
-import com.azure.storage.blob.perf.core.BlobTestBase;
+import com.azure.storage.blob.perf.core.AbstractUploadTest;
 import com.azure.storage.blob.specialized.BlobOutputStream;
 import reactor.core.publisher.Mono;
 
 import java.io.IOException;
 
 import static com.azure.perf.test.core.TestDataCreationHelper.writeBytesToOutputStream;
 
-public class UploadOutputStreamTest extends BlobTestBase<BlobPerfStressOptions> {
+public class UploadOutputStreamTest extends AbstractUploadTest<BlobPerfStressOptions> {
     public UploadOutputStreamTest(BlobPerfStressOptions options) {
         super(options);
     }

sdk/storage/azure-storage-perf/src/main/java/com/azure/storage/blob/perf/core/AbstractDownloadTest.java

@@ -0,0 +1,23 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.azure.storage.blob.perf.core;
+
+import com.azure.storage.blob.perf.BlobPerfStressOptions;
+import reactor.core.publisher.Mono;
+
+import static com.azure.perf.test.core.TestDataCreationHelper.createRandomByteBufferFlux;
+
+public abstract class AbstractDownloadTest <TOptions extends BlobPerfStressOptions> extends BlobTestBase<TOptions> {
+
+    public AbstractDownloadTest(TOptions options) {
+        super(options, BLOB_NAME_PREFIX);
+    }
+
+    // Upload one blob for the whole test run. All tests can download the same blob
+    public Mono<Void> globalSetupAsync() {
+        return super.globalSetupAsync()
+            .then(blobAsyncClient.upload(createRandomByteBufferFlux(options.getSize()), null))
+            .then();
+    }
+}

sdk/storage/azure-storage-perf/src/main/java/com/azure/storage/blob/perf/core/AbstractUploadTest.java

@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.azure.storage.blob.perf.core;
+
+import com.azure.storage.blob.perf.BlobPerfStressOptions;
+import reactor.core.publisher.Mono;
+
+import java.util.UUID;
+
+import static com.azure.perf.test.core.TestDataCreationHelper.createRandomByteBufferFlux;
+
+public abstract class AbstractUploadTest<TOptions extends BlobPerfStressOptions> extends BlobTestBase<TOptions> {
+
+    public AbstractUploadTest(TOptions options) {
+        super(options, BLOB_NAME_PREFIX + UUID.randomUUID());
+    }
+
+    @Override
+    public Mono<Void> setupAsync() {
+        // Upload one blob per test
+        return super.setupAsync()
+            .then(blobAsyncClient.upload(createRandomByteBufferFlux(options.getSize()), null))
+            .then();
+    }
+}