Merge pull request Azure#16 from alzimmermsft/AzStorage_Blobs_Verify_…

…Tests Fixed snapshot passing and compilation error
alzimmermsft · Jun 27, 2019 · db66a89 · db66a89
2 parents dc42213 + d1dcee2
commit db66a89
Show file tree

Hide file tree

Showing 12 changed files with 131 additions and 90 deletions.
diff --git a/storage/client/src/main/java/com/azure/storage/blob/AppendBlobClientBuilder.java b/storage/client/src/main/java/com/azure/storage/blob/AppendBlobClientBuilder.java
@@ -4,18 +4,17 @@
 package com.azure.storage.blob;
 
 import com.azure.core.credentials.TokenCredential;
-import com.azure.core.http.policy.BearerTokenAuthenticationPolicy;
-import com.azure.core.util.configuration.Configuration;
 import com.azure.core.http.HttpClient;
 import com.azure.core.http.HttpPipeline;
 import com.azure.core.http.policy.AddDatePolicy;
+import com.azure.core.http.policy.BearerTokenAuthenticationPolicy;
 import com.azure.core.http.policy.HttpLogDetailLevel;
 import com.azure.core.http.policy.HttpLoggingPolicy;
 import com.azure.core.http.policy.HttpPipelinePolicy;
 import com.azure.core.http.policy.RequestIdPolicy;
-import com.azure.core.http.policy.RetryPolicy;
 import com.azure.core.http.policy.UserAgentPolicy;
 import com.azure.core.implementation.util.ImplUtils;
+import com.azure.core.util.configuration.Configuration;
 import com.azure.core.util.configuration.ConfigurationManager;
 import com.azure.storage.blob.implementation.AzureBlobStorageBuilder;
 import com.azure.storage.common.credentials.SASTokenCredential;
@@ -105,7 +104,7 @@ private AzureBlobStorageBuilder buildImpl() {
             .build();
 
         return new AzureBlobStorageBuilder()
-            .url(String.format("%s/%s/%s", endpoint.toString(), containerName, blobName))
+            .url(String.format("%s/%s/%s", endpoint, containerName, blobName))
             .pipeline(pipeline);
     }
 
@@ -134,9 +133,15 @@ public AppendBlobClientBuilder endpoint(String endpoint) {
         try {
             url = new URL(endpoint);
             BlobURLParts parts = URLParser.parse(url);
-            this.endpoint = parts.scheme() + "://" + parts.host();;
-            this.containerName = parts.containerName();
-            this.blobName = parts.blobName();
+            this.endpoint = parts.scheme() + "://" + parts.host();
+
+            if (parts.containerName() != null) {
+                this.containerName = parts.containerName();
+            }
+
+            if (parts.blobName() != null) {
+                this.blobName = parts.blobName();
+            }
         } catch (MalformedURLException | UnknownHostException ex) {
             throw new IllegalArgumentException("The Azure Storage Blob endpoint url is malformed.");
         }

diff --git a/storage/client/src/main/java/com/azure/storage/blob/BlobAsyncRawClient.java b/storage/client/src/main/java/com/azure/storage/blob/BlobAsyncRawClient.java
@@ -240,7 +240,7 @@ public Mono<DownloadAsyncResponse> download(BlobRange range, BlobAccessCondition
         // TODO: range is BlobRange but expected as String
         // TODO: figure out correct response
         return postProcessResponse(this.azureBlobStorage.blobs().downloadWithRestResponseAsync(
-            null, null, null, null, null, range.toHeaderValue(), getMD5,
+            null, null, snapshot, null, null, range.toHeaderValue(), getMD5,
             null, null, null, null,
             accessConditions.leaseAccessConditions(),  accessConditions.modifiedAccessConditions(), Context.NONE))
             // Convert the autorest response to a DownloadAsyncResponse, which enable reliable download.

diff --git a/storage/client/src/main/java/com/azure/storage/blob/BlobClientBuilder.java b/storage/client/src/main/java/com/azure/storage/blob/BlobClientBuilder.java
@@ -12,7 +12,6 @@
 import com.azure.core.http.policy.HttpLoggingPolicy;
 import com.azure.core.http.policy.HttpPipelinePolicy;
 import com.azure.core.http.policy.RequestIdPolicy;
-import com.azure.core.http.policy.RetryPolicy;
 import com.azure.core.http.policy.UserAgentPolicy;
 import com.azure.core.implementation.util.ImplUtils;
 import com.azure.core.util.configuration.Configuration;
@@ -106,7 +105,7 @@ private AzureBlobStorageBuilder buildImpl() {
             .build();
 
         return new AzureBlobStorageBuilder()
-            .url(String.format("%s/%s/%s", endpoint.toString(), containerName, blobName))
+            .url(String.format("%s/%s/%s", endpoint, containerName, blobName))
             .pipeline(pipeline);
     }
 
@@ -135,10 +134,19 @@ public BlobClientBuilder endpoint(String endpoint) {
         try {
             url = new URL(endpoint);
             BlobURLParts parts = URLParser.parse(url);
-            this.endpoint = parts.scheme() + "://" + parts.host();;
-            this.containerName = parts.containerName();
-            this.blobName = parts.blobName();
-            this.snapshot = parts.snapshot();
+            this.endpoint = parts.scheme() + "://" + parts.host();
+
+            if (parts.containerName() != null) {
+                this.containerName = parts.containerName();
+            }
+
+            if (parts.blobName() != null) {
+                this.blobName = parts.blobName();
+            }
+
+            if (parts.snapshot() != null) {
+                this.snapshot = parts.snapshot();
+            }
         } catch (MalformedURLException | UnknownHostException ex) {
             throw new IllegalArgumentException("The Azure Storage Blob endpoint url is malformed.");
         }

diff --git a/storage/client/src/main/java/com/azure/storage/blob/BlockBlobClient.java b/storage/client/src/main/java/com/azure/storage/blob/BlockBlobClient.java
@@ -17,7 +17,6 @@
 import com.azure.storage.blob.models.SourceModifiedAccessConditions;
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.ByteBufAllocator;
-import io.netty.buffer.Unpooled;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 import reactor.core.scheduler.Schedulers;
@@ -188,7 +187,7 @@ public void uploadFromFile(String filePath) throws IOException {
 
     public void uploadFromFile(String filePath, BlobHTTPHeaders headers, Metadata metadata,
             BlobAccessConditions accessConditions, Duration timeout) throws IOException {
-        Mono<Void> upload = this.blockBlobAsyncClient.uploadFromFile(filePath, headers, metadata, accessConditions);
+        Mono<Void> upload = this.blockBlobAsyncClient.uploadFromFile(filePath, BlockBlobAsyncClient.BLOB_DEFAULT_UPLOAD_BLOCK_SIZE, headers, metadata, accessConditions);
 
         try {
             if (timeout == null) {
@@ -240,14 +239,22 @@ public VoidResponse stageBlock(String base64BlockID, InputStream data, long leng
      *         An optional timeout value beyond which a {@link RuntimeException} will be raised.
      */
     public VoidResponse stageBlock(String base64BlockID, InputStream data, long length,
-            LeaseAccessConditions leaseAccessConditions, Duration timeout) throws IOException {
+            LeaseAccessConditions leaseAccessConditions, Duration timeout) {
 
-        // buffer strategy for UX study only
-        byte[] bufferedData = new byte[(int)length];
-        data.read(bufferedData);
+        Flux<ByteBuf> fbb = Flux.range(0, (int) Math.ceil((double) length / (double) BlockBlobAsyncClient.BLOB_DEFAULT_UPLOAD_BLOCK_SIZE))
+            .map(i -> i * BlockBlobAsyncClient.BLOB_DEFAULT_UPLOAD_BLOCK_SIZE)
+            .concatMap(pos -> Mono.fromCallable(() -> {
+                long count = pos + BlockBlobAsyncClient.BLOB_DEFAULT_UPLOAD_BLOCK_SIZE > length ? length - pos : BlockBlobAsyncClient.BLOB_DEFAULT_UPLOAD_BLOCK_SIZE;
+                byte[] cache = new byte[(int) count];
+                int read = 0;
+                while (read < count) {
+                    read += data.read(cache, read, (int) count - read);
+                }
+                return ByteBufAllocator.DEFAULT.buffer((int) count).writeBytes(cache);
+            }));
 
         Mono<VoidResponse> response = blockBlobAsyncClient.stageBlock(base64BlockID,
-            Flux.just(Unpooled.wrappedBuffer(bufferedData)), length, leaseAccessConditions);
+            fbb.subscribeOn(Schedulers.elastic()), length, leaseAccessConditions);
         return Utility.blockWithOptionalTimeout(response, timeout);
     }
 

diff --git a/storage/client/src/main/java/com/azure/storage/blob/BlockBlobClientBuilder.java b/storage/client/src/main/java/com/azure/storage/blob/BlockBlobClientBuilder.java
@@ -12,7 +12,6 @@
 import com.azure.core.http.policy.HttpLoggingPolicy;
 import com.azure.core.http.policy.HttpPipelinePolicy;
 import com.azure.core.http.policy.RequestIdPolicy;
-import com.azure.core.http.policy.RetryPolicy;
 import com.azure.core.http.policy.UserAgentPolicy;
 import com.azure.core.implementation.util.ImplUtils;
 import com.azure.core.util.configuration.Configuration;
@@ -110,7 +109,7 @@ private AzureBlobStorageBuilder buildImpl() {
             .build();
 
         return new AzureBlobStorageBuilder()
-            .url(String.format("%s/%s/%s", endpoint.toString(), containerName, blobName))
+            .url(String.format("%s/%s/%s", endpoint, containerName, blobName))
             .pipeline(pipeline);
     }
 
@@ -139,10 +138,19 @@ public BlockBlobClientBuilder endpoint(String endpoint) {
         try {
             url = new URL(endpoint);
             BlobURLParts parts = URLParser.parse(url);
-            this.endpoint = parts.scheme() + "://" + parts.host();;
-            this.containerName = parts.containerName();
-            this.blobName = parts.blobName();
-            this.snapshot = parts.snapshot();
+            this.endpoint = parts.scheme() + "://" + parts.host();
+
+            if (parts.containerName() != null) {
+                this.containerName = parts.containerName();
+            }
+
+            if (parts.blobName() != null) {
+                this.blobName = parts.blobName();
+            }
+
+            if (parts.snapshot() != null) {
+                this.snapshot = parts.snapshot();
+            }
         } catch (MalformedURLException | UnknownHostException ex) {
             throw new IllegalArgumentException("The Azure Storage Blob endpoint url is malformed.");
         }

diff --git a/storage/client/src/main/java/com/azure/storage/blob/PageBlobClientBuilder.java b/storage/client/src/main/java/com/azure/storage/blob/PageBlobClientBuilder.java
@@ -12,7 +12,6 @@
 import com.azure.core.http.policy.HttpLoggingPolicy;
 import com.azure.core.http.policy.HttpPipelinePolicy;
 import com.azure.core.http.policy.RequestIdPolicy;
-import com.azure.core.http.policy.RetryPolicy;
 import com.azure.core.http.policy.UserAgentPolicy;
 import com.azure.core.implementation.util.ImplUtils;
 import com.azure.core.util.configuration.Configuration;
@@ -135,10 +134,19 @@ public PageBlobClientBuilder endpoint(String endpoint) {
         try {
             url = new URL(endpoint);
             BlobURLParts parts = URLParser.parse(url);
-            this.endpoint = parts.scheme() + "://" + parts.host();;
-            this.containerName = parts.containerName();
-            this.blobName = parts.blobName();
-            this.snapshot = parts.snapshot();
+            this.endpoint = parts.scheme() + "://" + parts.host();
+
+            if (parts.containerName() != null) {
+                this.containerName = parts.containerName();
+            }
+
+            if (parts.blobName() != null) {
+                this.blobName = parts.blobName();
+            }
+
+            if (parts.snapshot() != null) {
+                this.snapshot = parts.snapshot();
+            }
         } catch (MalformedURLException | UnknownHostException ex) {
             throw new IllegalArgumentException("The Azure Storage Blob endpoint url is malformed.");
         }

diff --git a/storage/client/src/main/java/com/azure/storage/blob/StorageClientBuilder.java b/storage/client/src/main/java/com/azure/storage/blob/StorageClientBuilder.java
@@ -141,7 +141,7 @@ public StorageClientBuilder endpoint(String endpoint) {
     }
 
     String endpoint() {
-        return this.endpoint.toString();
+        return this.endpoint;
     }
 
     /**

diff --git a/storage/client/src/main/java/com/azure/storage/blob/StorageException.java b/storage/client/src/main/java/com/azure/storage/blob/StorageException.java
@@ -6,7 +6,6 @@
 import com.azure.core.exception.HttpResponseException;
 import com.azure.storage.blob.models.StorageErrorCode;
 import com.azure.storage.blob.models.StorageErrorException;
-import reactor.core.publisher.Mono;
 
 /**
  * A {@code StorageException} is thrown whenever Azure Storage successfully returns an error code that is not 200-level.
@@ -17,9 +16,8 @@
  * perform a getProperties request on an entity to determine whether it exists or not. If it does not exists, an
  * exception will be thrown even though this may be considered an expected indication of absence in this case.
  *
- * @apiNote ## Sample Code \n
- * [!code-java[Sample_Code](../azure-storage-java/src/test/java/com/microsoft/azure/storage/Samples.java?name=exception "Sample code for StorageExceptions")] \n
- * For more samples, please see the [Samples file](%https://github.com/Azure/azure-storage-java/blob/master/src/test/java/com/microsoft/azure/storage/Samples.java)
+ * <p><strong>Sample Code</strong></p>
+ * <p>For more samples, please see the <a href="https://github.com/Azure/azure-storage-java/blob/master/src/test/java/com/microsoft/azure/storage/Samples.java">sample file</a></p>
  */
 public final class StorageException extends HttpResponseException {
 

diff --git a/storage/client/src/main/java/com/azure/storage/common/policy/RequestRetryOptions.java b/storage/client/src/main/java/com/azure/storage/common/policy/RequestRetryOptions.java
@@ -61,9 +61,9 @@ public RequestRetryOptions() {
      *         this field, make sure you understand the issues around reading stale and potentially-inconsistent data at
      *         <a href=https://docs.microsoft.com/en-us/azure/storage/common/storage-designing-ha-apps-with-ragrs>this webpage</a>
      *
-     * @apiNote ## Sample Code \n
-     * [!code-java[Sample_Code](../azure-storage-java/src/test/java/com/microsoft/azure/storage/Samples.java?name=pipeline_options "Sample code for RequestRetryOptions constructor")] \n
-     * For more samples, please see the [Samples file] (https://github.com/Azure/azure-storage-java/blob/master/src/test/java/com/microsoft/azure/storage/Samples.java)
+     * <p><strong>Sample Code</strong></p>
+     *
+     * <p>For more samples, please see the <a href="https://github.com/Azure/azure-storage-java/blob/master/src/test/java/com/microsoft/azure/storage/Samples.java">samples file</a></p>
      */
     public RequestRetryOptions(RetryPolicyType retryPolicyType, Integer maxTries, Integer tryTimeout,
             Long retryDelayInMs, Long maxRetryDelayInMs, String secondaryHost) {

diff --git a/storage/client/src/test/java/com/azure/storage/blob/BlobAPITest.groovy b/storage/client/src/test/java/com/azure/storage/blob/BlobAPITest.groovy
@@ -222,6 +222,26 @@ class BlobAPITest extends APISpec {
         thrown(StorageException)
     }
 
+    def "Download snapshot"() {
+        when:
+        ByteArrayOutputStream originalStream = new ByteArrayOutputStream()
+        bu.download(originalStream)
+
+        String snapshot = bu.createSnapshot().value()
+        BlockBlobClient bu2 = bu.asBlockBlobClient()
+        bu2.upload(new ByteArrayInputStream("ABC".getBytes()), 3)
+
+        then:
+        BlobClient bu3 = BlobClient.blobClientBuilder()
+            .endpoint(bu.getBlobUrl().toString())
+            .snapshot(snapshot)
+            .credential(primaryCreds)
+            .buildClient()
+        ByteArrayOutputStream snapshotStream = new ByteArrayOutputStream()
+        bu3.download(snapshotStream)
+        snapshotStream.toByteArray() == originalStream.toByteArray()
+    }
+
     /*def "Download context"() {
         setup:
         def pipeline = HttpPipeline.build(getStubFactory(getContextStubPolicy(206, BlobDownloadHeaders)))
@@ -280,14 +300,12 @@ class BlobAPITest extends APISpec {
     @Unroll
     def "Get properties AC"() {
         setup:
-        match = setupBlobMatchCondition(bu, match)
-        leaseID = setupBlobLeaseCondition(bu, leaseID)
         BlobAccessConditions bac = new BlobAccessConditions()
-            .leaseAccessConditions(new LeaseAccessConditions().leaseId(leaseID))
+            .leaseAccessConditions(new LeaseAccessConditions().leaseId(setupBlobLeaseCondition(bu, leaseID)))
             .modifiedAccessConditions(new ModifiedAccessConditions()
                 .ifModifiedSince(modified)
                 .ifUnmodifiedSince(unmodified)
-                .ifMatch(match)
+                .ifMatch(setupBlobMatchCondition(bu, match))
                 .ifNoneMatch(noneMatch))
 
         expect:
@@ -306,9 +324,8 @@ class BlobAPITest extends APISpec {
     @Unroll
     def "Get properties AC fail"() {
         setup:
-        setupBlobLeaseCondition(bu, leaseID)
         BlobAccessConditions bac = new BlobAccessConditions()
-            .leaseAccessConditions(new LeaseAccessConditions().leaseId(leaseID))
+            .leaseAccessConditions(new LeaseAccessConditions().leaseId(setupBlobLeaseCondition(bu, leaseID)))
             .modifiedAccessConditions(new ModifiedAccessConditions()
                 .ifModifiedSince(modified)
                 .ifUnmodifiedSince(unmodified)
@@ -356,7 +373,7 @@ class BlobAPITest extends APISpec {
 
     def "Set HTTP headers null"() {
         setup:
-        VoidResponse response = bu.setHTTPHeaders(null, null, null)
+        VoidResponse response = bu.setHTTPHeaders(null)
 
         expect:
         response.statusCode() == 200
@@ -366,7 +383,16 @@ class BlobAPITest extends APISpec {
     // TODO (alzimmer): Figure out why getProperties returns null after setHTTPHeaders
     /*def "Set HTTP headers min"() {
         when:
-        bu.setHTTPHeaders(new BlobHTTPHeaders().blobContentType("type"))
+        BlobProperties properties = bu.getProperties().value()
+        BlobHTTPHeaders headers = new BlobHTTPHeaders()
+            .blobContentEncoding(properties.contentEncoding())
+            .blobContentDisposition(properties.contentDisposition())
+            .blobContentType("type")
+            .blobCacheControl(properties.cacheControl())
+            .blobContentLanguage(properties.contentLanguage())
+            .blobContentMD5(Base64.getDecoder().decode(properties.contentMD5()))
+
+        bu.setHTTPHeaders(headers)
 
         then:
         bu.getProperties().headers().value("x-ms-blob-content-type") == "type"

diff --git a/storage/client/src/test/java/com/azure/storage/blob/BlockBlobAPITest.groovy b/storage/client/src/test/java/com/azure/storage/blob/BlockBlobAPITest.groovy
@@ -5,6 +5,7 @@ package com.azure.storage.blob
 
 import com.azure.core.http.HttpHeaders
 import com.azure.core.http.rest.Response
+import com.azure.core.http.rest.VoidResponse
 import com.azure.storage.blob.models.*
 import spock.lang.Unroll
 
@@ -26,7 +27,7 @@ class BlockBlobAPITest extends APISpec {
 
     def "Stage block"() {
         setup:
-        Response<BlockBlobItem> response = bu.stageBlock(getBlockID(), defaultInputStream.get(), defaultDataSize)
+        VoidResponse response = bu.stageBlock(getBlockID(), defaultInputStream.get(), defaultDataSize)
         HttpHeaders headers = response.headers()
 
         expect: