diff --git a/UPGRADING.md b/UPGRADING.md index 42cb15c3..db83da06 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -1,6 +1,12 @@ cos-php-sdk-v5 Upgrade Guide ==================== -2.0.6 to 2.0.7 + +2.0.8 to 2.0.9 +---------- +- Fix bug of `listObjectVersions` +- Update `getObject` with param of `saveas` + +2.0.7 to 2.0.8 ---------- - Fix presigned url when using tmpSecretId/tmpSecretKey/Token diff --git a/src/Qcloud/Cos/Client.php b/src/Qcloud/Cos/Client.php index 54af3394..f0bd21b8 100644 --- a/src/Qcloud/Cos/Client.php +++ b/src/Qcloud/Cos/Client.php @@ -21,7 +21,7 @@ class Client extends GuzzleClient { - const VERSION = '2.0.8'; + const VERSION = '2.0.9'; public $httpClient; @@ -49,6 +49,7 @@ public function __construct($cosConfig) { $this->cosConfig['endpoint'] = isset($cosConfig['endpoint']) ? $cosConfig['endpoint'] : 'myqcloud.com'; $this->cosConfig['domain'] = isset($cosConfig['domain']) ? $cosConfig['domain'] : null; $this->cosConfig['proxy'] = isset($cosConfig['proxy']) ? $cosConfig['proxy'] : null; + $this->cosConfig['retry'] = isset($cosConfig['retry']) ? $cosConfig['retry'] : 1; $this->cosConfig['userAgent'] = isset($cosConfig['userAgent']) ? $cosConfig['userAgent'] : 'cos-php-sdk-v5.'. Client::VERSION; $this->cosConfig['pathStyle'] = isset($cosConfig['pathStyle']) ? $cosConfig['pathStyle'] : false; @@ -113,14 +114,20 @@ public function __destruct() { } public function __call($method, array $args) { - try { - return parent::__call(ucfirst($method), $args); - } catch (CommandException $e) { - $previous = $e->getPrevious(); - if ($previous !== null) { - throw $previous; - } else { - throw $e; + for ($i = 1; $i <= $this->cosConfig['retry']; $i++) { + try { + return parent::__call(ucfirst($method), $args); + } catch (CommandException $e) { + if ($i != $this->cosConfig['retry']) { + sleep(1 << ($i-1)); + continue; + } + $previous = $e->getPrevious(); + if ($previous !== null) { + throw $previous; + } else { + throw $e; + } } } } @@ -137,13 +144,13 @@ private function createPresignedUrl(RequestInterface $request, $expires) { return $this->signature->createPresignedUrl($request, $expires); } - public function getPresignetUrl($method, $args, $expires = null) { + public function getPresignetUrl($method, $args, $expires = "+30 minutes") { $command = $this->getCommand($method, $args); $request = $this->commandToRequestTransformer($command); return $this->createPresignedUrl($request, $expires); } - public function getObjectUrl($bucket, $key, $expires = null, array $args = array()) { + public function getObjectUrl($bucket, $key, $expires = "+30 minutes", array $args = array()) { $command = $this->getCommand('GetObject', $args + array('Bucket' => $bucket, 'Key' => $key)); $request = $this->commandToRequestTransformer($command); return $this->createPresignedUrl($request, $expires)->__toString(); @@ -151,7 +158,7 @@ public function getObjectUrl($bucket, $key, $expires = null, array $args = array public function upload($bucket, $key, $body, $options = array()) { $body = Psr7\stream_for($body); - $options['PartSize'] = isset($options['PartSize']) ? $options['PartSize'] : MultipartUpload::MIN_PART_SIZE; + $options['PartSize'] = isset($options['PartSize']) ? $options['PartSize'] : MultipartUpload::DEFAULT_PART_SIZE; if ($body->getSize() < $options['PartSize']) { $rt = $this->putObject(array( 'Bucket' => $bucket, diff --git a/src/Qcloud/Cos/CommandToRequestTransformer.php b/src/Qcloud/Cos/CommandToRequestTransformer.php index e39b7a0c..7d2bb46c 100644 --- a/src/Qcloud/Cos/CommandToRequestTransformer.php +++ b/src/Qcloud/Cos/CommandToRequestTransformer.php @@ -130,6 +130,7 @@ public function metadataTransformer(CommandInterface $command, $request) { $request = $request->withHeader('x-cos-meta-' . $key, $value); } } + $request = headersMap($command, $request); return $request; } diff --git a/src/Qcloud/Cos/Common.php b/src/Qcloud/Cos/Common.php index c188002e..e962ed47 100644 --- a/src/Qcloud/Cos/Common.php +++ b/src/Qcloud/Cos/Common.php @@ -33,3 +33,16 @@ function endWith($haystack, $needle) { } return (substr($haystack, -$length) === $needle); } + +function headersMap($command, $request) { + $headermap = array( + 'TransferEncoding'=>'Transfer-Encoding', + 'ChannelId'=>'x-cos-channel-id' + ); + foreach ($headermap as $key => $value) { + if (isset($command[$key])) { + $request = $request->withHeader($value, $command[$key]); + } + } + return $request; +} diff --git a/src/Qcloud/Cos/Copy.php b/src/Qcloud/Cos/Copy.php index 98ed075d..661df592 100644 --- a/src/Qcloud/Cos/Copy.php +++ b/src/Qcloud/Cos/Copy.php @@ -6,9 +6,6 @@ use GuzzleHttp\Pool; class Copy { - /** - * const var: part size from 1MB to 5GB, and max parts of 10000 are allowed for each upload. - */ const MIN_PART_SIZE = 1048576; const MAX_PART_SIZE = 5368709120; const DEFAULT_PART_SIZE = 52428800; @@ -79,7 +76,7 @@ public function uploadParts($uploadId) { 'CopySource'=> $copySourcePath, 'CopySourceRange' => 'bytes='.((string)$offset).'-'.(string)($offset+$partSize - 1), ); - if(!isset($parts[$partNumber])) { + if(!isset($this->parts[$partNumber])) { $command = $this->client->getCommand('uploadPartCopy', $params); $request = $this->client->commandToRequestTransformer($command); $this->commandList[$index] = $command; @@ -103,14 +100,14 @@ public function uploadParts($uploadId) { }, 'rejected' => function ($reason, $index) { + $index = $index += 1; $retry = 2; for ($i = 1; $i <= $retry; $i++) { - $index = $index += 1; try { - $rt =$this->client->execute($commandList[$index]); + $rt =$this->client->execute($this->commandList[$index]); $part = array('PartNumber' => $index, 'ETag' => $rt['ETag']); $this->parts[$index] = $part; - } catch(Exception $e) { + } catch(\Exception $e) { if ($i == $retry) { throw($e); } diff --git a/src/Qcloud/Cos/MultipartUpload.php b/src/Qcloud/Cos/MultipartUpload.php index 94cc5bbd..604ffe4d 100644 --- a/src/Qcloud/Cos/MultipartUpload.php +++ b/src/Qcloud/Cos/MultipartUpload.php @@ -3,65 +3,94 @@ namespace Qcloud\Cos; use Qcloud\Cos\Exception\CosException; +use GuzzleHttp\Pool; class MultipartUpload { - /** - * const var: part size from 1MB to 5GB, and max parts of 10000 are allowed for each upload. - */ const MIN_PART_SIZE = 1048576; const MAX_PART_SIZE = 5368709120; const DEFAULT_PART_SIZE = 52428800; const MAX_PARTS = 10000; private $client; - private $body; private $options; private $partSize; + private $parts; + private $body; public function __construct($client, $body, $options = array()) { - $this->client = $client; + $minPartSize = $options['PartSize']; + unset($options['PartSize']); $this->body = $body; + $this->client = $client; $this->options = $options; - $this->partSize = $this->calculatePartSize($options['PartSize']); - unset($options['PartSize']); + $this->partSize = $this->calculatePartSize($minPartSize); + $this->concurrency = isset($options['Concurrency']) ? $options['Concurrency'] : 10; + $this->parts = []; + $this->partNumberList = []; } - public function performUploading() { - $rt = $this->initiateMultipartUpload(); - $uploadId = $rt['UploadId']; - $partNumber = 1; - $parts = array(); - for (;;) { - if ($this->body->eof()) { - break; - } - $body = $this->body->read($this->partSize); - if (empty($body)) { - break; + $uploadId= $this->initiateMultipartUpload(); + $this->uploadParts($uploadId); + foreach ( $this->parts as $key => $row ){ + $num1[$key] = $row ['PartNumber']; + $num2[$key] = $row ['ETag']; + } + array_multisort($num1, SORT_ASC, $num2, SORT_ASC, $this->parts); + return $this->client->completeMultipartUpload(array( + 'Bucket' => $this->options['Bucket'], + 'Key' => $this->options['Key'], + 'UploadId' => $uploadId, + 'Parts' => $this->parts) + ); + + } + public function uploadParts($uploadId) { + $uploadRequests = function ($uploadId) { + $partNumber = 1; + $index = 1; + for ( ; ; $partNumber ++) { + if ($this->body->eof()) { + break; + } + $body = $this->body->read($this->partSize); + if (empty($body)) { + break; + } + if (isset($this->parts[$partNumber])) { + continue; + } + $this->partNumberList[$index] = $partNumber; + $params = array( + 'Bucket' => $this->options['Bucket'], + 'Key' => $this->options['Key'], + 'UploadId' => $uploadId, + 'PartNumber' => $partNumber, + 'Body' => $body + ); + if(!isset($this->parts[$partNumber])) { + $command = $this->client->getCommand('uploadPart', $params); + $request = $this->client->commandToRequestTransformer($command); + $index ++; + yield $request; + } } - $result = $this->client->uploadPart(array( - 'Bucket' => $this->options['Bucket'], - 'Key' => $this->options['Key'], - 'Body' => $body, - 'UploadId' => $uploadId, - 'PartNumber' => $partNumber)); - if (md5($body) != substr($result['ETag'], 1, -1)){ - throw new CosException("ETag check inconsistency"); + }; + $pool = new Pool($this->client->httpClient, $uploadRequests($uploadId), [ + 'concurrency' => $this->concurrency, + 'fulfilled' => function ($response, $index) { + $index = $index + 1; + $partNumber = $this->partNumberList[$index]; + $etag = $response->getHeaders()["ETag"][0]; + $part = array('PartNumber' => $partNumber, 'ETag' => $etag); + $this->parts[$partNumber] = $part; + }, + + 'rejected' => function ($reason, $index) { + throw($reason); } - $part = array('PartNumber' => $partNumber, 'ETag' => $result['ETag']); - array_push($parts, $part); - ++$partNumber; - } - try { - $rt = $this->client->completeMultipartUpload(array( - 'Bucket' => $this->options['Bucket'], - 'Key' => $this->options['Key'], - 'UploadId' => $uploadId, - 'Parts' => $parts)); - } catch(\Exception $e){ - throw $e; - } - return $rt; + ]); + $promise = $pool->promise(); + $promise->wait(); } public function resumeUploading() { @@ -73,54 +102,35 @@ public function resumeUploading() { $parts = array(); if (count($rt['Parts']) > 0) { foreach ($rt['Parts'] as $part) { - $parts[$part['PartNumber'] - 1] = array('PartNumber' => $part['PartNumber'], 'ETag' => $part['ETag']); + $this->parts[$part['PartNumber']] = array('PartNumber' => $part['PartNumber'], 'ETag' => $part['ETag']); } } - for ($partNumber = 1;;++$partNumber) { - if ($this->body->eof()) { - break; - } - $body = $this->body->read($this->partSize); - - if (array_key_exists($partNumber-1, $parts)){ - - if (md5($body) != substr($parts[$partNumber-1]['ETag'], 1, -1)){ - throw new CosException("ETag check inconsistency"); - } - continue; - } - - $result = $this->client->uploadPart(array( - 'Bucket' => $this->options['Bucket'], - 'Key' => $this->options['Key'], - 'Body' => $body, - 'UploadId' => $uploadId, - 'PartNumber' => $partNumber)); - if (md5($body) != substr($result['ETag'], 1, -1)){ - throw new CosException("ETag check inconsistency"); - } - $parts[$partNumber-1] = array('PartNumber' => $partNumber, 'ETag' => $result['ETag']); - + $this->uploadParts($uploadId); + foreach ( $this->parts as $key => $row ){ + $num1[$key] = $row ['PartNumber']; + $num2[$key] = $row ['ETag']; } - $rt = $this->client->completeMultipartUpload(array( + array_multisort($num1, SORT_ASC, $num2, SORT_ASC, $this->parts); + return $this->client->completeMultipartUpload(array( 'Bucket' => $this->options['Bucket'], 'Key' => $this->options['Key'], 'UploadId' => $uploadId, - 'Parts' => $parts)); - return $rt; + 'Parts' => $this->parts) + ); } - private function calculatePartSize($minPartSize) { + private function calculatePartSize($minPartSize) + { $partSize = intval(ceil(($this->body->getSize() / self::MAX_PARTS))); $partSize = max($minPartSize, $partSize); $partSize = min($partSize, self::MAX_PART_SIZE); $partSize = max($partSize, self::MIN_PART_SIZE); - return $partSize; } private function initiateMultipartUpload() { $result = $this->client->createMultipartUpload($this->options); - return $result; + return $result['UploadId']; } + } diff --git a/src/Qcloud/Cos/ResultTransformer.php b/src/Qcloud/Cos/ResultTransformer.php index 02afc84c..7764e427 100644 --- a/src/Qcloud/Cos/ResultTransformer.php +++ b/src/Qcloud/Cos/ResultTransformer.php @@ -33,7 +33,15 @@ public function writeDataToLocal(CommandInterface $command, RequestInterface $re if ($action == "GetObject") { if (isset($command['SaveAs'])) { $fp = fopen($command['SaveAs'], "wb"); - fwrite($fp, $response->getBody()); + $stream = $response->getBody(); + $offset = 0; + $partsize = 8192; + while (!$stream->eof()) { + $output = $stream->read($partsize); + fseek($fp, $offset); + fwrite($fp, $output); + $offset += $partsize; + } fclose($fp); } } diff --git a/src/Qcloud/Cos/Service.php b/src/Qcloud/Cos/Service.php index ffd3a3e0..2b805a3e 100644 --- a/src/Qcloud/Cos/Service.php +++ b/src/Qcloud/Cos/Service.php @@ -3430,13 +3430,17 @@ public static function getService() { 'type' => 'object', 'properties' => array( 'Status' => array( - 'type' => 'string'), + 'type' => 'string' + ), 'Name' => array( - 'type' => 'string'), + 'type' => 'string' + ), 'Type' => array( - 'type' => 'string'), + 'type' => 'string' + ), 'ForcedReplacement' => array( - 'type' => 'string'), + 'type' => 'string' + ), ), ), ), @@ -4454,10 +4458,9 @@ public static function getService() { 'type' => 'string', 'location' => 'xml', ), - 'Versions' => array( + 'Version' => array( 'type' => 'array', 'location' => 'xml', - 'sentAs' => 'Versions', 'data' => array( 'xmlFlattened' => true, ), diff --git a/src/Qcloud/Cos/Signature.php b/src/Qcloud/Cos/Signature.php index a9e03930..72e3ecbb 100644 --- a/src/Qcloud/Cos/Signature.php +++ b/src/Qcloud/Cos/Signature.php @@ -20,6 +20,9 @@ public function signRequest(RequestInterface $request) { return $request->withHeader('Authorization', $authorization); } public function createAuthorization(RequestInterface $request, $expires = "+30 minutes") { + if (is_null($expires)) { + $expires = "+30 minutes"; + } $signTime = (string)(time() - 60) . ';' . (string)(strtotime($expires)); $httpString = strtolower($request->getMethod()) . "\n" . urldecode($request->getUri()->getPath()) . "\n\nhost=" . $request->getHeader("Host")[0]. "\n"; diff --git a/src/Qcloud/Cos/Tests/Test.php b/src/Qcloud/Cos/Tests/COSTest.php similarity index 99% rename from src/Qcloud/Cos/Tests/Test.php rename to src/Qcloud/Cos/Tests/COSTest.php index e78724a3..b0ad6d7a 100644 --- a/src/Qcloud/Cos/Tests/Test.php +++ b/src/Qcloud/Cos/Tests/COSTest.php @@ -68,7 +68,6 @@ public function testValidRegionBucket() $regionlist = array('cn-east','ap-shanghai', 'cn-south','ap-guangzhou', 'cn-north','ap-beijing-1', - 'cn-south-2','ap-guangzhou-2', 'cn-southwest','ap-chengdu', 'sg','ap-singapore', 'tj','ap-beijing-1',