From 6669e01e09c1ac4a841b78e59fea144a74abfb0a Mon Sep 17 00:00:00 2001 From: xushiwei Date: Thu, 4 Jul 2013 17:59:24 +0800 Subject: [PATCH 1/4] x --- qiniu/io.php | 6 +- qiniu/resumable_io.php | 132 +++++++++++++++++++++++++++++++++++++++++ qiniu/rs.php | 5 +- qiniu/rs_utils.php | 4 +- 4 files changed, 141 insertions(+), 6 deletions(-) create mode 100644 qiniu/resumable_io.php diff --git a/qiniu/io.php b/qiniu/io.php index 8c5a7488..34952bb4 100644 --- a/qiniu/io.php +++ b/qiniu/io.php @@ -14,7 +14,7 @@ class Qiniu_PutExtra public $CheckCrc = 0; } -function Qiniu_Put($upToken, $key, $body, $putExtra) // => ($data, $err) +function Qiniu_Put($upToken, $key, $body, $fsize, $putExtra) // => ($putRet, $err) { global $QINIU_UP_HOST; @@ -39,7 +39,7 @@ function Qiniu_Put($upToken, $key, $body, $putExtra) // => ($data, $err) return Qiniu_Client_CallWithMultipartForm($client, $QINIU_UP_HOST, $fields, $files); } -function Qiniu_PutFile($upToken, $key, $localFile, $putExtra) // => ($data, $err) +function Qiniu_PutFile($upToken, $key, $localFile, $putExtra) // => ($putRet, $err) { global $QINIU_UP_HOST; @@ -67,3 +67,5 @@ function Qiniu_PutFile($upToken, $key, $localFile, $putExtra) // => ($data, $err return Qiniu_Client_CallWithForm($client, $QINIU_UP_HOST, $fields, 'multipart/form-data'); } +// ---------------------------------------------------------- + diff --git a/qiniu/resumable_io.php b/qiniu/resumable_io.php new file mode 100644 index 00000000..ee7bfcfb --- /dev/null +++ b/qiniu/resumable_io.php @@ -0,0 +1,132 @@ + $blockCnt +{ + return ($fsize + (QINIU_RIO_BLOCK_SIZE - 1)) >> QINIU_RIO_BLOCK_BITS; +} + +// ---------------------------------------------------------- +// internal func Qiniu_Rio_Mkblock/Mkfile + +function Qiniu_Rio_Mkblock($self, $host, $reader, $size) // => ($blkputRet, $err) +{ + $body = fread($reader, $size); + if ($body === false) { + $err = Qiniu_NewError(0, 'fread failed'); + return array(null, $err); + } + if (strlen($body) != $fsize) { + $err = Qiniu_NewError(0, 'fread failed: unexpected eof'); + return array(null, $err); + } + + $url = $host . '/mkblk/' . $size; + return Qiniu_Client_CallWithForm($self, $url, $body, 'application/octet-stream'); +} + +function Qiniu_Rio_Mkfile($self, $host, $key, $fsize, $extra) // => ($putRet, $err) +{ + $entry = $extra->Bucket . ':' . $key; + $url = $host . '/rs-mkfile/' . Qiniu_Encode($entry) . '/fsize/' . $fsize; + + if (!empty($extra->MimeType)) { + $url .= '/mimeType/' . Qiniu_Encode($extra->MimeType); + } + + $ctxs = array(); + for ($extra->Progresses as $prog) { + $ctxs []= $prog['ctx']; + } + $body = implode(',', $ctxs); + + return Qiniu_Client_CallWithForm($self, $url, $body, 'text/plain'); +} + +// ---------------------------------------------------------- +// class Qiniu_Rio_UploadClient + +class Qiniu_Rio_UploadClient +{ + public $uptoken; + + public function __construct($uptoken) + { + $this->uptoken = $uptoken; + } + + public function RoundTrip($req) // => ($resp, $error) + { + $token = $this->uptoken; + $req->Header['Authorization'] = "UpToken $token"; + return Qiniu_Client_do($req); + } +} + +// ---------------------------------------------------------- +// class Qiniu_Rio_Put/PutFile + +function Qiniu_Rio_Put($upToken, $key, $body, $fsize, $putExtra) // => ($putRet, $err) +{ + global $QINIU_UP_HOST; + + $self = new Qiniu_Rio_UploadClient($upToken); + + $progresses = array(); + $host = $QINIU_UP_HOST; + $uploaded = 0; + while($uploaded < $fsize) { + if ($uploaded + QINIU_RIO_BLOCK_SIZE < $fsize) { + $bsize = $fsize - $uploaded; + } else { + $bsize = QINIU_RIO_BLOCK_SIZE; + } + list($blkputRet, $err) = Qiniu_Rio_Mkblock($self, $host, $body, $bsize); + $host = $blkputRet['host']; + $uploaded += $bsize; + $progresses []= $blkputRet; + } + + return Qiniu_Rio_Mkfile($self, $host, $key, $fsize, $putExtra); +} + +function Qiniu_Rio_PutFile($upToken, $key, $localFile, $putExtra) // => ($putRet, $err) +{ + + $fp = fopen($localFile, 'rb'); + if ($fp === false) { + $err = Qiniu_NewError(0, 'fopen failed'); + return array(null, $err); + } + + $fi = fstat($fp); + + Qiniu_Rio_Put($upToken, $key, $fp, $fi['size'], $putExtra); + fclose($fp); +} + +// ---------------------------------------------------------- + diff --git a/qiniu/rs.php b/qiniu/rs.php index c1471412..579c533e 100644 --- a/qiniu/rs.php +++ b/qiniu/rs.php @@ -170,9 +170,8 @@ function Qiniu_RS_Copy($self, $bucketSrc, $keySrc, $bucketDest, $keyDest) // => return Qiniu_Client_CallNoRet($self, $QINIU_RS_HOST . $uri); } - // ---------------------------------------------------------- -//batch +// batch function Qiniu_RS_Batch($self, $ops) // => ($data, $error) { @@ -222,3 +221,5 @@ function Qiniu_RS_BatchCopy($self, $entryPairs) return Qiniu_RS_Batch($self, $params); } +// ---------------------------------------------------------- + diff --git a/qiniu/rs_utils.php b/qiniu/rs_utils.php index 79421bae..e8f47b80 100644 --- a/qiniu/rs_utils.php +++ b/qiniu/rs_utils.php @@ -3,14 +3,14 @@ require_once("rs.php"); require_once("io.php"); -function Qiniu_RS_Put($self, $bucket, $key, $body, $putExtra) // => ($data, $err) +function Qiniu_RS_Put($self, $bucket, $key, $body, $putExtra) // => ($putRet, $err) { $putPolicy = new Qiniu_RS_PutPolicy("$bucket:$key"); $upToken = $putPolicy->Token($self->Mac); return Qiniu_Put($upToken, $key, $body, $putExtra); } -function Qiniu_RS_PutFile($self, $bucket, $key, $localFile, $putExtra) // => ($data, $err) +function Qiniu_RS_PutFile($self, $bucket, $key, $localFile, $putExtra) // => ($putRet, $err) { $putPolicy = new Qiniu_RS_PutPolicy("$bucket:$key"); $upToken = $putPolicy->Token($self->Mac); From a89f7fedfb51883428ae3a17aac765887099d264 Mon Sep 17 00:00:00 2001 From: xushiwei Date: Thu, 4 Jul 2013 20:29:45 +0800 Subject: [PATCH 2/4] Rio support --- qiniu/resumable_io.php | 29 +++++++++---- qiniu/rs_utils.php | 1 + tests/RioTest.php | 93 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 tests/RioTest.php diff --git a/qiniu/resumable_io.php b/qiniu/resumable_io.php index ee7bfcfb..112efd8d 100644 --- a/qiniu/resumable_io.php +++ b/qiniu/resumable_io.php @@ -16,6 +16,10 @@ class Qiniu_Rio_PutExtra public $Progresses = null; // 可选。上传进度：[]BlkputRet public $Notify = null; // 进度通知：func(blkIdx int, blkSize int, ret *BlkputRet) public $NotifyErr = null; // 错误通知：func(blkIdx int, blkSize int, err error) + + public function __construct($bucket = null) { + $this->Bucket = $bucket; + } } // ---------------------------------------------------------- @@ -34,12 +38,19 @@ function Qiniu_Rio_BlockCount($fsize) // => $blockCnt function Qiniu_Rio_Mkblock($self, $host, $reader, $size) // => ($blkputRet, $err) { - $body = fread($reader, $size); - if ($body === false) { - $err = Qiniu_NewError(0, 'fread failed'); - return array(null, $err); + if (is_resource($reader)) { + $body = fread($reader, $size); + if ($body === false) { + $err = Qiniu_NewError(0, 'fread failed'); + return array(null, $err); + } + } else { + list($body, $err) = $reader->Read($size); + if ($err !== null) { + return array(null, $err); + } } - if (strlen($body) != $fsize) { + if (strlen($body) != $size) { $err = Qiniu_NewError(0, 'fread failed: unexpected eof'); return array(null, $err); } @@ -58,7 +69,7 @@ function Qiniu_Rio_Mkfile($self, $host, $key, $fsize, $extra) // => ($putRet, $e } $ctxs = array(); - for ($extra->Progresses as $prog) { + foreach ($extra->Progresses as $prog) { $ctxs []= $prog['ctx']; } $body = implode(',', $ctxs); @@ -98,8 +109,8 @@ function Qiniu_Rio_Put($upToken, $key, $body, $fsize, $putExtra) // => ($putRet, $progresses = array(); $host = $QINIU_UP_HOST; $uploaded = 0; - while($uploaded < $fsize) { - if ($uploaded + QINIU_RIO_BLOCK_SIZE < $fsize) { + while ($uploaded < $fsize) { + if ($fsize < $uploaded + QINIU_RIO_BLOCK_SIZE) { $bsize = $fsize - $uploaded; } else { $bsize = QINIU_RIO_BLOCK_SIZE; @@ -110,12 +121,12 @@ function Qiniu_Rio_Put($upToken, $key, $body, $fsize, $putExtra) // => ($putRet, $progresses []= $blkputRet; } + $putExtra->Progresses = $progresses; return Qiniu_Rio_Mkfile($self, $host, $key, $fsize, $putExtra); } function Qiniu_Rio_PutFile($upToken, $key, $localFile, $putExtra) // => ($putRet, $err) { - $fp = fopen($localFile, 'rb'); if ($fp === false) { $err = Qiniu_NewError(0, 'fopen failed'); diff --git a/qiniu/rs_utils.php b/qiniu/rs_utils.php index e8f47b80..f2877e5a 100644 --- a/qiniu/rs_utils.php +++ b/qiniu/rs_utils.php @@ -2,6 +2,7 @@ require_once("rs.php"); require_once("io.php"); +require_once("resumable_io.php"); function Qiniu_RS_Put($self, $bucket, $key, $body, $putExtra) // => ($putRet, $err) { diff --git a/tests/RioTest.php b/tests/RioTest.php new file mode 100644 index 00000000..4dd6931b --- /dev/null +++ b/tests/RioTest.php @@ -0,0 +1,93 @@ +off = $off; + } + + public function Read($bytes) // => ($data, $err) + { + $off = $this->off; + $data = ''; + for ($i = 0; $i < $bytes; $i++) { + $data .= chr(65 + ($off % 26)); // ord('A') = 65 + $off++; + } + $this->off = $off; + return array($data, null); + } +} + +class RioTest extends PHPUnit_Framework_TestCase +{ + public $bucket; + public $client; + + public function setUp() + { + initKeys(); + $this->client = new Qiniu_MacHttpClient(null); + $this->bucket = getenv("QINIU_BUCKET_NAME"); + } + + public function testMockReader() + { + $reader = new MockReader; + list($data) = $reader->Read(5); + $this->assertEquals($data, "ABCDE"); + + list($data) = $reader->Read(27); + $this->assertEquals($data, "FGHIJKLMNOPQRSTUVWXYZABCDEF"); + } + + public function testPut() + { + $key = 'testRioPut' . getTid(); + $err = Qiniu_RS_Delete($this->client, $this->bucket, $key); + + $putPolicy = new Qiniu_RS_PutPolicy($this->bucket); + $upToken = $putPolicy->Token(null); + $putExtra = new Qiniu_Rio_PutExtra($this->bucket); + $reader = new MockReader; + list($ret, $err) = Qiniu_Rio_Put($upToken, $key, $reader, 5, $putExtra); + $this->assertNull($err); + $this->assertEquals($ret['hash'], "Fnvgeq9GDVk6Mj0Nsz2gW2S_3LOl"); + var_dump($ret); + + list($ret, $err) = Qiniu_RS_Stat($this->client, $this->bucket, $key); + $this->assertNull($err); + var_dump($ret); + + $err = Qiniu_RS_Delete($this->client, $this->bucket, $key); + $this->assertNull($err); + } + + public function testLargePut() + { + $key = 'testRioLargePut' . getTid(); + $err = Qiniu_RS_Delete($this->client, $this->bucket, $key); + + $putPolicy = new Qiniu_RS_PutPolicy($this->bucket); + $upToken = $putPolicy->Token(null); + $putExtra = new Qiniu_Rio_PutExtra($this->bucket); + $reader = new MockReader; + list($ret, $err) = Qiniu_Rio_Put($upToken, $key, $reader, QINIU_RIO_BLOCK_SIZE + 5, $putExtra); + $this->assertNull($err); + $this->assertEquals($ret['hash'], "lgQEOCZ8Ievliq8XOfZmWTndgOll"); + var_dump($ret); + + list($ret, $err) = Qiniu_RS_Stat($this->client, $this->bucket, $key); + $this->assertNull($err); + var_dump($ret); + + $err = Qiniu_RS_Delete($this->client, $this->bucket, $key); + $this->assertNull($err); + } +} + From ecb08f8f4829b31b4c70047fe5dcf228949292f6 Mon Sep 17 00:00:00 2001 From: xushiwei Date: Thu, 4 Jul 2013 21:01:46 +0800 Subject: [PATCH 3/4] rio testcase --- qiniu/io.php | 2 +- qiniu/resumable_io.php | 4 ++-- qiniu/rs_utils.php | 24 ++++++++++++++++++++++++ tests/RioTest.php | 22 ---------------------- tests/RsUtilsTest.php | 39 ++++++++++++++++++++++++++++++++++++++- tests/bootstrap.php | 22 ++++++++++++++++++++++ 6 files changed, 87 insertions(+), 26 deletions(-) diff --git a/qiniu/io.php b/qiniu/io.php index 34952bb4..10df1b24 100644 --- a/qiniu/io.php +++ b/qiniu/io.php @@ -14,7 +14,7 @@ class Qiniu_PutExtra public $CheckCrc = 0; } -function Qiniu_Put($upToken, $key, $body, $fsize, $putExtra) // => ($putRet, $err) +function Qiniu_Put($upToken, $key, $body, $putExtra) // => ($putRet, $err) { global $QINIU_UP_HOST; diff --git a/qiniu/resumable_io.php b/qiniu/resumable_io.php index 112efd8d..7e477a73 100644 --- a/qiniu/resumable_io.php +++ b/qiniu/resumable_io.php @@ -134,9 +134,9 @@ function Qiniu_Rio_PutFile($upToken, $key, $localFile, $putExtra) // => ($putRet } $fi = fstat($fp); - - Qiniu_Rio_Put($upToken, $key, $fp, $fi['size'], $putExtra); + $result = Qiniu_Rio_Put($upToken, $key, $fp, $fi['size'], $putExtra); fclose($fp); + return $result; } // ---------------------------------------------------------- diff --git a/qiniu/rs_utils.php b/qiniu/rs_utils.php index f2877e5a..15343d62 100644 --- a/qiniu/rs_utils.php +++ b/qiniu/rs_utils.php @@ -18,3 +18,27 @@ function Qiniu_RS_PutFile($self, $bucket, $key, $localFile, $putExtra) // => ($p return Qiniu_PutFile($upToken, $key, $localFile, $putExtra); } +function Qiniu_RS_Rput($self, $bucket, $key, $body, $fsize, $putExtra) // => ($putRet, $err) +{ + $putPolicy = new Qiniu_RS_PutPolicy("$bucket:$key"); + $upToken = $putPolicy->Token($self->Mac); + if ($putExtra == null) { + $putExtra = new Qiniu_Rio_PutExtra($bucket); + } else { + $putExtra->Bucket = $bucket; + } + return Qiniu_Rio_Put($upToken, $key, $body, $fsize, $putExtra); +} + +function Qiniu_RS_RputFile($self, $bucket, $key, $localFile, $putExtra) // => ($putRet, $err) +{ + $putPolicy = new Qiniu_RS_PutPolicy("$bucket:$key"); + $upToken = $putPolicy->Token($self->Mac); + if ($putExtra == null) { + $putExtra = new Qiniu_Rio_PutExtra($bucket); + } else { + $putExtra->Bucket = $bucket; + } + return Qiniu_Rio_PutFile($upToken, $key, $localFile, $putExtra); +} + diff --git a/tests/RioTest.php b/tests/RioTest.php index 4dd6931b..1e195b04 100644 --- a/tests/RioTest.php +++ b/tests/RioTest.php @@ -2,28 +2,6 @@ require_once("bootstrap.php"); -class MockReader -{ - private $off = 0; - - public function __construct($off = 0) - { - $this->off = $off; - } - - public function Read($bytes) // => ($data, $err) - { - $off = $this->off; - $data = ''; - for ($i = 0; $i < $bytes; $i++) { - $data .= chr(65 + ($off % 26)); // ord('A') = 65 - $off++; - } - $this->off = $off; - return array($data, null); - } -} - class RioTest extends PHPUnit_Framework_TestCase { public $bucket; diff --git a/tests/RsUtilsTest.php b/tests/RsUtilsTest.php index dded1764..6b8123a5 100644 --- a/tests/RsUtilsTest.php +++ b/tests/RsUtilsTest.php @@ -14,6 +14,43 @@ public function setUp() $this->bucket = getenv("QINIU_BUCKET_NAME"); } + public function testRput() + { + $key = 'tmp/testRput' . getTid(); + $err = Qiniu_RS_Delete($this->client, $this->bucket, $key); + + $reader = new MockReader; + list($ret, $err) = Qiniu_RS_Rput($this->client, $this->bucket, $key, $reader, 5, null); + $this->assertNull($err); + var_dump($ret); + $this->assertEquals($ret['hash'], "Fnvgeq9GDVk6Mj0Nsz2gW2S_3LOl"); + + list($ret, $err) = Qiniu_RS_Stat($this->client, $this->bucket, $key); + $this->assertNull($err); + var_dump($ret); + + $err = Qiniu_RS_Delete($this->client, $this->bucket, $key); + $this->assertNull($err); + } + + public function testRputFile() + { + $key = 'tmp/testRputFile' . getTid(); + $err = Qiniu_RS_Delete($this->client, $this->bucket, $key); + + list($ret, $err) = Qiniu_RS_RputFile($this->client, $this->bucket, $key, __file__, null); + $this->assertNull($err); + var_dump($ret); + $this->assertArrayHasKey('hash', $ret); + + list($ret, $err) = Qiniu_RS_Stat($this->client, $this->bucket, $key); + $this->assertNull($err); + var_dump($ret); + + $err = Qiniu_RS_Delete($this->client, $this->bucket, $key); + $this->assertNull($err); + } + public function testPutFile() { $key = 'tmp/testPutFile' . getTid(); @@ -39,7 +76,7 @@ public function testPut() $key = 'tmp/testPut' . getTid(); $err = Qiniu_RS_Delete($this->client, $this->bucket, $key); - list($ret, $err) = Qiniu_RS_Put($this->client, $this->bucket, $key, "hello world!", null); + list($ret, $err) = Qiniu_RS_Put($this->client, $this->bucket, $key, 'hello world!', null); $this->assertNull($err); $this->assertArrayHasKey('hash', $ret); var_dump($ret); diff --git a/tests/bootstrap.php b/tests/bootstrap.php index ddfe8a56..3757bfac 100644 --- a/tests/bootstrap.php +++ b/tests/bootstrap.php @@ -24,3 +24,25 @@ function getTid() { return $tid; } +class MockReader +{ + private $off = 0; + + public function __construct($off = 0) + { + $this->off = $off; + } + + public function Read($bytes) // => ($data, $err) + { + $off = $this->off; + $data = ''; + for ($i = 0; $i < $bytes; $i++) { + $data .= chr(65 + ($off % 26)); // ord('A') = 65 + $off++; + } + $this->off = $off; + return array($data, null); + } +} + From 34852818aa853cea91d4a16a2273f9112b2451c3 Mon Sep 17 00:00:00 2001 From: xushiwei Date: Thu, 4 Jul 2013 21:13:48 +0800 Subject: [PATCH 4/4] Update CHANGELOG.md --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 575950e3..dfe2ebf8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ ## CHANGE LOG +### v6.1.1 + +2013-07-04 issue [#24](https://github.com/qiniu/api/pull/24) + +- 支持断点续上传(`Qiniu_RS_Rput`, `Qiniu_RS_RputFile`) + + ### v6.1.0 2013-07-04 issue [#22](https://github.com/qiniu/api/pull/22)