From 755eee551305c630cd03db2cfa5111cf7b9274dc Mon Sep 17 00:00:00 2001 From: Josh Friedman Date: Thu, 22 Jun 2017 10:54:14 -0700 Subject: [PATCH 1/3] Adding line about exceptions to changelog --- ChangeLog.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog.txt b/ChangeLog.txt index d64351f5a41a4..920ca54f303a4 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -1,5 +1,6 @@ 2017.06.21 Version 5.3.1 * Fixed a bug in specific upload case for block blobs. This only affects uploads greater than the max put blob threshold, that have increased the streamWriteSizeInBytes beyond the 4 MB and storeBlobContentMD5 has been disabled. + * In some cases in the above mentioned upload path, fixed a bug where StorageExceptions were being thrown instead of IOExceptions. 2017.06.13 Version 5.3.0 * Fixed a bug where the transactional MD5 check would fail when downloading a range of blob or file and the recovery action is performed on a subsection of the range. From 98b2d83fc50b28833c0bffa34bea5e8bd7e55ad7 Mon Sep 17 00:00:00 2001 From: Josh Friedman Date: Tue, 27 Jun 2017 16:22:42 -0700 Subject: [PATCH 2/3] Add error receiving response event --- ChangeLog.txt | 3 + .../azure/storage/EventFiringTests.java | 103 +++++++++++++++++- .../storage/ErrorReceivingResponseEvent.java | 38 +++++++ .../azure/storage/OperationContext.java | 68 +++++++++++- .../azure/storage/core/ExecutionEngine.java | 100 +++++++++-------- .../azure/storage/core/LogConstants.java | 1 + 6 files changed, 262 insertions(+), 51 deletions(-) create mode 100644 microsoft-azure-storage/src/com/microsoft/azure/storage/ErrorReceivingResponseEvent.java diff --git a/ChangeLog.txt b/ChangeLog.txt index 920ca54f303a4..06394b55e4607 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -1,3 +1,6 @@ +2017.XX.XX Version X.X.X + * Added ErrorReceivingResponseEvent which fires when a network error occurs before the responseReceivedEvent fires. If the responseReceivedEvent fires sucessfully, this new event will not fire. + 2017.06.21 Version 5.3.1 * Fixed a bug in specific upload case for block blobs. This only affects uploads greater than the max put blob threshold, that have increased the streamWriteSizeInBytes beyond the 4 MB and storeBlobContentMD5 has been disabled. * In some cases in the above mentioned upload path, fixed a bug where StorageExceptions were being thrown instead of IOExceptions. diff --git a/microsoft-azure-storage-test/src/com/microsoft/azure/storage/EventFiringTests.java b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/EventFiringTests.java index 4ea007be63c96..6ee2236d86e86 100644 --- a/microsoft-azure-storage-test/src/com/microsoft/azure/storage/EventFiringTests.java +++ b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/EventFiringTests.java @@ -14,18 +14,20 @@ */ package com.microsoft.azure.storage; -import com.microsoft.azure.storage.blob.BlobRequestOptions; -import com.microsoft.azure.storage.blob.CloudBlobClient; -import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.*; import com.microsoft.azure.storage.core.SR; import com.microsoft.azure.storage.TestRunners.CloudTests; import com.microsoft.azure.storage.TestRunners.DevFabricTests; import com.microsoft.azure.storage.TestRunners.DevStoreTests; +import org.apache.http.protocol.HTTP; import org.junit.Test; import org.junit.experimental.categories.Category; +import java.io.ByteArrayInputStream; +import java.io.IOException; import java.net.HttpURLConnection; +import java.net.SocketException; import java.net.URISyntaxException; import java.util.ArrayList; @@ -111,6 +113,22 @@ public void eventOccurred(ResponseReceivedEvent eventArg) { } }); + eventContext.getErrorReceivingResponseEventHandler().addListener(new StorageEvent() { + + @Override + public void eventOccurred(ErrorReceivingResponseEvent eventArg) { + fail("This event should not trigger"); + } + }); + + OperationContext.getGlobalErrorReceivingResponseEventHandler().addListener(new StorageEvent() { + + @Override + public void eventOccurred(ErrorReceivingResponseEvent eventArg) { + fail("This event should not trigger"); + } + }); + assertEquals(0, callList.size()); assertEquals(0, globalCallList.size()); @@ -138,6 +156,85 @@ public void eventOccurred(ResponseReceivedEvent eventArg) { assertEquals(2, globalCallList.size()); } + @Test + public void testErrorReceivingResponseEvent() throws URISyntaxException, StorageException { + final ArrayList callList = new ArrayList(); + final ArrayList globalCallList = new ArrayList(); + + OperationContext eventContext = new OperationContext(); + BlobRequestOptions options = new BlobRequestOptions(); + options.setRetryPolicyFactory(new RetryNoRetry()); + + // setting the sending request event handler to trigger an exception. + // this is a retryable exception + eventContext.getSendingRequestEventHandler().addListener(new StorageEvent() { + @Override + public void eventOccurred(SendingRequestEvent eventArg) { + HttpURLConnection connection = (HttpURLConnection) eventArg.getConnectionObject(); + connection.setFixedLengthStreamingMode(0); + } + }); + + eventContext.getErrorReceivingResponseEventHandler().addListener(new StorageEvent() { + @Override + public void eventOccurred(ErrorReceivingResponseEvent eventArg) { + assertEquals(eventArg.getRequestResult(), eventArg.getOpContext().getLastResult()); + callList.add(true); + } + }); + + OperationContext.getGlobalErrorReceivingResponseEventHandler().addListener(new StorageEvent() { + @Override + public void eventOccurred(ErrorReceivingResponseEvent eventArg) { + assertEquals(eventArg.getRequestResult(), eventArg.getOpContext().getLastResult()); + globalCallList.add(true); + } + }); + + CloudBlobClient blobClient = TestHelper.createCloudBlobClient(); + CloudBlobContainer container = blobClient.getContainerReference("container1"); + container.createIfNotExists(); + + try { + CloudBlockBlob blob1 = container.getBlockBlobReference("blob1"); + try { + String blockID = String.format("%08d", 1); + blob1.uploadBlock(blockID, BlobTestHelper.getRandomDataStream(10), 10, null, options, eventContext); + } catch (Exception e) { } + + // make sure both the local and globab context update + assertEquals(1, callList.size()); + assertEquals(1, globalCallList.size()); + + // make sure only global updates by replacing the local with a no-op event + eventContext + .setErrorReceivingResponseEventHandler(new StorageEventMultiCaster>()); + try { + String blockID2 = String.format("%08d", 2); + blob1.uploadBlock(blockID2, BlobTestHelper.getRandomDataStream(10), 10, null, options, eventContext); + } catch (Exception e) { } + + assertEquals(1, callList.size()); + assertEquals(2, globalCallList.size()); + + // make sure global does not update by replacing the global with a no-op + OperationContext + .setGlobalErrorReceivingResponseEventHandler(new StorageEventMultiCaster>()); + + // make sure neither update + try { + String blockID3 = String.format("%08d", 3); + blob1.uploadBlock(blockID3, BlobTestHelper.getRandomDataStream(10), 10, null, options, eventContext); + } catch (Exception e) { } + + assertEquals(1, callList.size()); + assertEquals(2, globalCallList.size()); + } + finally { + container.deleteIfExists(); + } + } + @Test public void testRequestCompletedEvents() throws URISyntaxException, StorageException { final ArrayList callList = new ArrayList(); diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/ErrorReceivingResponseEvent.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/ErrorReceivingResponseEvent.java new file mode 100644 index 0000000000000..84faacaa55765 --- /dev/null +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/ErrorReceivingResponseEvent.java @@ -0,0 +1,38 @@ +/** + * Copyright Microsoft Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.microsoft.azure.storage; + +/** + * Represents an event that is fired when a network error occurs before the HTTP response status and headers are received. + */ +public final class ErrorReceivingResponseEvent extends BaseEvent { + + /** + * Creates an instance of the BaseEvent class that is fired when a network error occurs before the HTTP response status and headers are received. + * + * @param opContext + * An {@link OperationContext} object that represents the context for the current operation. This object + * is used to track requests to the storage service, and to provide additional runtime information about + * the operation. + * @param connectionObject + * Represents a connection object. Currently only java.net.HttpURLConnection is supported as + * a connection object. + * @param requestResult + * A {@link RequestResult} object that represents the current request result. + */ + public ErrorReceivingResponseEvent(OperationContext opContext, Object connectionObject, RequestResult requestResult) { + super(opContext, connectionObject, requestResult); + } +} \ No newline at end of file diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/OperationContext.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/OperationContext.java index a59534d9f8dc5..0564fd01cee9a 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/OperationContext.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/OperationContext.java @@ -83,7 +83,8 @@ public final class OperationContext { private HashMap userHeaders; /** - * Represents an event that is triggered before sending a request. + * Represents an event that is triggered before sending a + * request. * * @see StorageEvent * @see StorageEventMultiCaster @@ -92,15 +93,23 @@ public final class OperationContext { private static StorageEventMultiCaster> globalSendingRequestEventHandler = new StorageEventMultiCaster>(); /** - * Represents an event that is triggered when a response is received from the storage service while processing a - * request. - * + * Represents an event that is triggered when a response is received from the storage service while processing a request + * * @see StorageEvent * @see StorageEventMultiCaster * @see ResponseReceivedEvent */ private static StorageEventMultiCaster> globalResponseReceivedEventHandler = new StorageEventMultiCaster>(); + /** + * Represents an event that is triggered when a network error occurs before the HTTP response status and headers are received. + * + * @see StorageEvent + * @see StorageEventMultiCaster + * @see ErrorReceivingResponseEvent + */ + private static StorageEventMultiCaster> globalErrorReceivingResponseEventHandler = new StorageEventMultiCaster>(); + /** * Represents an event that is triggered when a response received from the service is fully processed. * @@ -138,6 +147,15 @@ public final class OperationContext { */ private StorageEventMultiCaster> responseReceivedEventHandler = new StorageEventMultiCaster>(); + /** + * Represents an event that is triggered when a network error occurs before the HTTP response status and headers are received. + * + * @see StorageEvent + * @see StorageEventMultiCaster + * @see ErrorReceivingResponseEvent + */ + private StorageEventMultiCaster> errorReceivingResponseEventHandler = new StorageEventMultiCaster>(); + /** * Represents an event that is triggered when a response received from the service is fully processed. * @@ -285,6 +303,16 @@ public static StorageEventMultiCasterglobabErrorReceivingResponseEventHandler. + */ + public static StorageEventMultiCaster> getGlobalErrorReceivingResponseEventHandler() { + return OperationContext.globalErrorReceivingResponseEventHandler; + } + /** * Gets a global event multi-caster that is triggered when a request is completed. It allows event listeners to be * dynamically added and removed. @@ -325,6 +353,16 @@ public StorageEventMultiCastererrorReceivingResponseEventHandler. + */ + public StorageEventMultiCaster> getErrorReceivingResponseEventHandler() { + return this.errorReceivingResponseEventHandler; + } + /** * Gets an event multi-caster that is triggered when a request is completed. It allows event listeners to be * dynamically added and removed. @@ -450,6 +488,17 @@ public static void setGlobalResponseReceivedEventHandler( OperationContext.globalResponseReceivedEventHandler = globalResponseReceivedEventHandler; } + /** + * Sets a global event multi-caster that is triggered when a network error occurs before the HTTP response status and headers are received. + * + * @param globalErrorReceivingResponseEventHandler + * The {@link StorageEventMultiCaster} object to set for the globalErrorReceivingResponseEventHandler. + */ + public static void setGlobalErrorReceivingResponseEventHandler( + final StorageEventMultiCaster> globalErrorReceivingResponseEventHandler) { + OperationContext.globalErrorReceivingResponseEventHandler = globalErrorReceivingResponseEventHandler; + } + /** * Sets a global event multi-caster that is triggered when a request is completed. * @@ -494,6 +543,17 @@ public void setResponseReceivedEventHandler( this.responseReceivedEventHandler = responseReceivedEventHandler; } + /** + * Sets an event multi-caster that is triggered when a network error occurs before the HTTP response status and headers are received. + * + * @param errorReceivingResponseEventHandler + * The {@link StorageEventMultiCaster} object to set for the errorReceivingResponseEventHandler. + */ + public void setErrorReceivingResponseEventHandler( + final StorageEventMultiCaster> errorReceivingResponseEventHandler) { + this.errorReceivingResponseEventHandler = errorReceivingResponseEventHandler; + } + /** * Sets an event multi-caster that is triggered when a request is completed. * diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/core/ExecutionEngine.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/core/ExecutionEngine.java index 919f7811d7942..06cf8c8e03baf 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/core/ExecutionEngine.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/core/ExecutionEngine.java @@ -22,22 +22,7 @@ import java.util.Map.Entry; import java.util.concurrent.TimeoutException; -import com.microsoft.azure.storage.Constants; -import com.microsoft.azure.storage.LocationMode; -import com.microsoft.azure.storage.OperationContext; -import com.microsoft.azure.storage.RequestCompletedEvent; -import com.microsoft.azure.storage.RequestResult; -import com.microsoft.azure.storage.ResponseReceivedEvent; -import com.microsoft.azure.storage.RetryContext; -import com.microsoft.azure.storage.RetryInfo; -import com.microsoft.azure.storage.RetryNoRetry; -import com.microsoft.azure.storage.RetryPolicy; -import com.microsoft.azure.storage.RetryPolicyFactory; -import com.microsoft.azure.storage.RetryingEvent; -import com.microsoft.azure.storage.SendingRequestEvent; -import com.microsoft.azure.storage.StorageErrorCodeStrings; -import com.microsoft.azure.storage.StorageException; -import com.microsoft.azure.storage.StorageLocation; +import com.microsoft.azure.storage.*; /** * RESERVED FOR INTERNAL USE. A class that handles execution of StorageOperations and enforces retry policies. @@ -98,41 +83,55 @@ public static RESULT_TYPE executeWithRet request.getRequestProperty(Constants.HeaderConstants.DATE)); // 5. Potentially upload data - if (task.getSendStream() != null) { - Logger.info(opContext, LogConstants.UPLOAD); - final StreamMd5AndLength descriptor = Utility.writeToOutputStream(task.getSendStream(), - request.getOutputStream(), task.getLength(), false /* rewindStream */, - false /* calculate MD5 */, opContext, task.getRequestOptions()); - - task.validateStreamWrite(descriptor); - Logger.info(opContext, LogConstants.UPLOADDONE); - } + boolean responseReceivedEventTriggered = false; + try { + if (task.getSendStream() != null) { + Logger.info(opContext, LogConstants.UPLOAD); + final StreamMd5AndLength descriptor = Utility.writeToOutputStream(task.getSendStream(), + request.getOutputStream(), task.getLength(), false /* rewindStream */, + false /* calculate MD5 */, opContext, task.getRequestOptions()); + + task.validateStreamWrite(descriptor); + Logger.info(opContext, LogConstants.UPLOADDONE); + } + + Utility.logHttpRequest(request, opContext); - Utility.logHttpRequest(request, opContext); + // 6. Process the request - Get response + RequestResult currResult = task.getResult(); + currResult.setStartDate(new Date()); - // 6. Process the request - Get response - RequestResult currResult = task.getResult(); - currResult.setStartDate(new Date()); + Logger.info(opContext, LogConstants.GET_RESPONSE); - Logger.info(opContext, LogConstants.GET_RESPONSE); + currResult.setStatusCode(request.getResponseCode()); + currResult.setStatusMessage(request.getResponseMessage()); + currResult.setStopDate(new Date()); - currResult.setStatusCode(request.getResponseCode()); - currResult.setStatusMessage(request.getResponseMessage()); - currResult.setStopDate(new Date()); + currResult.setServiceRequestID(BaseResponse.getRequestId(request)); + currResult.setEtag(BaseResponse.getEtag(request)); + currResult.setRequestDate(BaseResponse.getDate(request)); + currResult.setContentMD5(BaseResponse.getContentMD5(request)); - currResult.setServiceRequestID(BaseResponse.getRequestId(request)); - currResult.setEtag(BaseResponse.getEtag(request)); - currResult.setRequestDate(BaseResponse.getDate(request)); - currResult.setContentMD5(BaseResponse.getContentMD5(request)); + // 7. Fire ResponseReceived Event + responseReceivedEventTriggered = true; + ExecutionEngine.fireResponseReceivedEvent(opContext, request, task.getResult()); - // 7. Fire ResponseReceived Event - ExecutionEngine.fireResponseReceivedEvent(opContext, request, task.getResult()); + Logger.info(opContext, LogConstants.RESPONSE_RECEIVED, currResult.getStatusCode(), + currResult.getServiceRequestID(), currResult.getContentMD5(), currResult.getEtag(), + currResult.getRequestDate()); - Logger.info(opContext, LogConstants.RESPONSE_RECEIVED, currResult.getStatusCode(), - currResult.getServiceRequestID(), currResult.getContentMD5(), currResult.getEtag(), - currResult.getRequestDate()); - - Utility.logHttpResponse(request, opContext); + Utility.logHttpResponse(request, opContext); + } + finally { + Logger.info(opContext, LogConstants.ERROR_RECEIVING_RESPONSE); + if (!responseReceivedEventTriggered) { + if (task.getResult().getStartDate() == null) { + task.getResult().setStartDate(new Date()); + } + + ExecutionEngine.fireErrorReceivingResponseEvent(opContext, request, task.getResult()); + } + } // 8. Pre-process response to check if there was an exception. Do Response parsing (headers etc). Logger.info(opContext, LogConstants.PRE_PROCESS); @@ -376,6 +375,19 @@ private static void fireResponseReceivedEvent(OperationContext opContext, HttpUR } } + /** + * Fires events representing that an error occurred when receiving the response. + */ + private static void fireErrorReceivingResponseEvent(OperationContext opContext, HttpURLConnection request, + RequestResult result) { + if (opContext.getErrorReceivingResponseEventHandler().hasListeners() + || OperationContext.getGlobalErrorReceivingResponseEventHandler().hasListeners()) { + ErrorReceivingResponseEvent event = new ErrorReceivingResponseEvent(opContext, request, result); + opContext.getErrorReceivingResponseEventHandler().fireEvent(event); + OperationContext.getGlobalErrorReceivingResponseEventHandler().fireEvent(event); + } + } + /** * Fires events representing that a response received from the service is fully processed. */ diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/core/LogConstants.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/core/LogConstants.java index 55fd5be15c245..a29cc3c4430a5 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/core/LogConstants.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/core/LogConstants.java @@ -22,6 +22,7 @@ public class LogConstants { public static final String COMPLETE = "Operation completed."; public static final String DO_NOT_RETRY_POLICY = "Retry policy did not allow for a retry. Failing. Error Message = '%s'."; public static final String DO_NOT_RETRY_TIMEOUT = "Operation cannot be retried because maximum execution timeout has been reached. Failing. Inner error Message = '%s'."; + public static final String ERROR_RECEIVING_RESPONSE = "A network error occurred before the HTTP response status and headers were received."; public static final String GET_RESPONSE = "Waiting for response."; public static final String INIT_LOCATION = "Starting operation with location '%s' per location mode '%s'."; public static final String NEXT_LOCATION = "The next location has been set to '%s', per location mode '%s'."; From ecde918aecc770f1fa2b73d22b88f82073cd1e33 Mon Sep 17 00:00:00 2001 From: Josh Friedman Date: Fri, 7 Jul 2017 10:09:06 -0700 Subject: [PATCH 3/3] Eight TB Test --- .../storage/blob/CloudPageBlobTests.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudPageBlobTests.java b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudPageBlobTests.java index 9dd0e6b265f0e..3282a1e6a9c15 100644 --- a/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudPageBlobTests.java +++ b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudPageBlobTests.java @@ -1200,4 +1200,36 @@ else if (overload == 2) { assertNotNull(copy.properties.getCopyState().getCopyDestinationSnapshotID()); assertNotNull(copy.getCopyState().getCompletionTime()); } + + @Test + public void testEightTBBlob() throws StorageException, URISyntaxException, IOException { + CloudPageBlob blob = this.container.getPageBlobReference("blob1"); + CloudPageBlob blob2 = this.container.getPageBlobReference("blob1"); + + long eightTb = 8L * 1024L * 1024L * 1024L * 1024L; + blob.create(eightTb); + assertEquals(eightTb, blob.getProperties().getLength()); + + blob2.downloadAttributes(); + assertEquals(eightTb, blob2.getProperties().getLength()); + + for (ListBlobItem listBlob : this.container.listBlobs()) { + CloudPageBlob listPageBlob = (CloudPageBlob)listBlob; + assertEquals(eightTb, listPageBlob.getProperties().getLength()); + } + + CloudPageBlob blob3 = this.container.getPageBlobReference("blob3"); + blob3.create(1024); + blob3.resize(eightTb); + + final Random randGenerator = new Random(); + final byte[] buffer = new byte[1024]; + randGenerator.nextBytes(buffer); + blob.uploadPages(new ByteArrayInputStream(buffer), eightTb - 512L, 512L); + + ArrayList ranges = blob.downloadPageRanges(); + assertEquals(1, ranges.size()); + assertEquals(eightTb - 512L, ranges.get(0).getStartOffset()); + assertEquals(eightTb - 1L, ranges.get(0).getEndOffset()); + } }