Merge pull request aws#566 from aws/retries-for-complete-mup-errors

Prevents S3 CompleteMultipartUpload failures by retrying the request.

src/Aws/S3/IncompleteMultipartUploadChecker.php

@@ -0,0 +1,56 @@
+<?php
+
+namespace Aws\S3;
+
+use Guzzle\Http\Exception\HttpException;
+use Guzzle\Http\Message\RequestInterface;
+use Guzzle\Http\Message\EntityEnclosingRequestInterface;
+use Guzzle\Http\Message\Response;
+use Guzzle\Plugin\Backoff\BackoffStrategyInterface;
+use Guzzle\Plugin\Backoff\AbstractBackoffStrategy;
+
+/**
+ * Retries CompleteMultipartUpload requests in the case of failure.
+ *
+ * From the S3 API Documentation:
+ *
+ *     Processing of a Complete Multipart Upload request could take several
+ *     minutes to complete. After Amazon S3 begins processing the request, it
+ *     sends an HTTP response header that specifies a 200 OK response. While
+ *     processing is in progress, Amazon S3 periodically sends whitespace
+ *     characters to keep the connection from timing out. Because a request
+ *     could fail after the initial 200 OK response has been sent, it is
+ *     important that you check the response body to determine whether the
+ *     request succeeded. Note that if Complete Multipart Upload fails,
+ *     applications should be prepared to retry the failed requests.
+ */
+class IncompleteMultipartUploadChecker extends AbstractBackoffStrategy
+{
+    public function __construct(BackoffStrategyInterface $next = null)
+    {
+        if ($next) {
+            $this->setNext($next);
+        }
+    }
+
+    public function makesDecision()
+    {
+        return true;
+    }
+
+    protected function getDelay(
+        $retries,
+        RequestInterface $request,
+        Response $response = null,
+        HttpException $e = null
+    ) {
+        if ($response && $request->getMethod() === 'POST'
+            && $request instanceof EntityEnclosingRequestInterface
+            && $response->getStatusCode() == 200
+            && strpos($request->getBody(), '<CompleteMultipartUpload xmlns') !== false
+            && strpos($response->getBody(), '<CompleteMultipartUploadResult xmlns') === false
+        ) {
+            return true;
+        }
+    }
+}

src/Aws/S3/S3Client.php

@@ -245,11 +245,13 @@ private static function createBackoffPlugin(S3ExceptionParser $exceptionParser)
     {
         return new BackoffPlugin(
             new TruncatedBackoffStrategy(3,
-                new CurlBackoffStrategy(null,
-                    new HttpBackoffStrategy(null,
-                        new SocketTimeoutChecker(
-                            new ExpiredCredentialsChecker($exceptionParser,
-                                new ExponentialBackoffStrategy()
+                new IncompleteMultipartUploadChecker(
+                    new CurlBackoffStrategy(null,
+                        new HttpBackoffStrategy(null,
+                            new SocketTimeoutChecker(
+                                new ExpiredCredentialsChecker($exceptionParser,
+                                    new ExponentialBackoffStrategy()
+                                )
                             )
                         )
                     )

tests/Aws/Tests/S3/S3ClientTest.php

@@ -326,7 +326,6 @@ public function testUploadsLargerObjectsUsingMultipartUploads()
         $this->setMockResponse($client, array(
             's3/initiate_multipart_upload',
             's3/upload_part',
-            's3/upload_part',
             's3/complete_multipart_upload'
         ));
         $history = new HistoryPlugin();
@@ -495,4 +494,23 @@ public function testUsesSigV4SignatureInSpecificRegions()
         $s3 = S3Client::factory(array(Options::REGION => 'cn-north-1'));
         $this->assertInstanceOf('Aws\S3\S3SignatureV4', $s3->getSignature());
     }
+
+    /**
+     * @covers Aws\S3\IncompleteMultipartUploadChecker
+     */
+    public function testHandlesDelayedErrorsInCompleteMultipartUpload()
+    {
+        $client = $this->getServiceBuilder()->get('s3', true);
+        $this->setMockResponse($client, array(
+            's3/complete_multipart_upload_error',
+            's3/complete_multipart_upload'
+        ));
+        $result = $client->completeMultipartUpload(array(
+            'Bucket'   => 'foo',
+            'Key'      => 'bar',
+            'UploadId' => 'baz',
+            'Parts'    => array()
+        ));
+        $this->assertTrue(isset($result['Location']));
+    }
 }

tests/mock/s3/complete_multipart_upload_error

@@ -0,0 +1,20 @@
+HTTP/1.1 200 OK
+x-amz-id-2: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+x-amz-request-id: XXXXXXXXXXXXXXXX
+Date: Mon, 30 Jul 2012 20:01:57 GMT
+Content-Type: application/xml
+Server: AmazonS3
+
+<?xml version="1.0" encoding="UTF-8"?>
+
+
+
+
+
+
+<Error>
+  <Code>InternalError</Code>
+  <Message>We encountered an internal error. Please try again.</Message>
+  <RequestId>656c76696e6727732072657175657374</RequestId>
+  <HostId>Uuag1LuByRx9e6j5Onimru9pO4ZVKnJ2Qz7/C1NPcfTWAtRPfTaOFg==</HostId>
+</Error>