Permalink
Browse files

Merge pull request #1 from ikbear/feature/copy_and_move_file

Feature/copy and move file
  • Loading branch information...
2 parents 16fc512 + 1ce48f9 commit 0f0c840648fa35b6a6de09f5b6e6c14be6c1ee60 @ikbear committed Jan 18, 2013
Showing with 323 additions and 90 deletions.
  1. +4 −2 .gitignore
  2. +9 −1 NEWS.md → CHANGELOG.md
  3. +3 −1 demo/rs_demo.js
  4. +1 −1 docs/README.md
  5. +67 −18 lib/auth.js
  6. +130 −18 lib/rs.js
  7. +9 −21 lib/util.js
  8. +2 −2 package.json
  9. +5 −0 test/config.js
  10. +93 −26 test/rs.test.js
View
@@ -17,5 +17,7 @@ results
node_modules
npm-debug.log
-demo/test.jpg
-test/config.js
+demo/*.jpg
+demo/rtest.js
+demo/itest.js
+demo/put.sh
View
@@ -1,10 +1,18 @@
#CHANGELOG
+## v2.3.2
+
+2012-12-31
+
+updated:
+
+- 修复crc32编码
+- 修复使用UploadToken方式上传时流式上传bug,流式上传不检查crc32
+
## v2.3.0
2012-11-23
updated:
- 启用新的 uploadToken(上传凭证)上传方式,可由客户方业务服务器生成上传凭证。上传前无需请求七牛云存储,减少http请求。
-- 增加断点续上传支持,并行断点传输速度更快
View
@@ -1,5 +1,7 @@
-var qiniu = require('../index.js');
+var fs = require('fs');
+var path = require('path');
var mime = require('mime');
+var qiniu = require('../index.js');
qiniu.conf.ACCESS_KEY = '<Please apply your access key>';
qiniu.conf.SECRET_KEY = '<Dont send your secret key to anyone>';
View
@@ -467,7 +467,7 @@ options
"crop": <ImageSizeAndOffsetGeometry>,
"quality": <ImageQuality>,
"rotate": <RotateDegree>,
- "format": <DestinationImageFormat>, =jpg, gif, png, tif, etc.
+ "format": <DestinationImageFormat>, = jpg, gif, png, tif, etc.
"auto_orient": <TrueOrFalse>
}
View
@@ -2,37 +2,86 @@ var crypto = require('crypto');
var config = require("./conf.js");
var util = require('./util.js');
+exports.UploadToken = UploadToken;
+exports.DownloadToken = DownloadToken;
+
+// ------------------------------------------------------------------------------------------
+// func generateSignature
+
+function generateSignature(params) {
+ var paramsString = JSON.stringify(params);
+ return util.encode(paramsString);
+}
+
+// ------------------------------------------------------------------------------------------
+// func generateEncodedDigest
+
+function generateEncodedDigest(signature) {
+ var hmac = crypto.createHmac('sha1', config.SECRET_KEY);
+ hmac.update(signature);
+ var digest = hmac.digest('base64');
+ return util.base64ToUrlsafe(digest);
+}
+
+// ------------------------------------------------------------------------------------------
+// func generateToken
+
+function generateToken(params) {
+ var signature = generateSignature(params);
+ var encodedDigest = generateEncodedDigest(signature);
+ return config.ACCESS_KEY + ":" + encodedDigest + ":" + signature;
+}
+
+// ------------------------------------------------------------------------------------------
+// type UploadToken
+
function UploadToken(opts) {
this.scope = opts.scope || null;
this.expires = opts.expires || 3600;
this.callbackUrl = opts.callbackUrl || null;
this.callbackBodyType = opts.callbackBodyType || null;
this.customer = opts.customer || null;
+ this.escape = opts.escape || 0;
+ this.asyncOps = opts.asyncOps || null;
}
-UploadToken.prototype.generateSignature = function() {
+UploadToken.prototype.generateToken = function() {
var params = {
- "scope": this.scope,
"deadline": this.expires + Math.floor(Date.now() / 1000),
- "callbackUrl": this.callbackUrl,
- "callbackBodyType": this.callbackBodyType,
- "customer": this.customer,
+ "escape": this.excape,
};
- var paramsString = JSON.stringify(params)
- return util.encode(paramsString);
+ if (this.scope !== null) {
+ params["scope"] = this.scope;
+ }
+ if (this.callbackUrl !== null) {
+ params["callbackurl"] = this.callbackUrl;
+ }
+ if (this.callbackBodyType !== null) {
+ params["callbackBodyType"] = this.callbackBodyType;
+ }
+ if (this.customer !== null) {
+ params["customer"] = this.customer;
+ }
+ if (this.asyncOps !== null) {
+ params["asyncOps"] = this.asyncOps;
+ }
+ return generateToken(params);
};
-UploadToken.prototype.generateEncodedDigest = function(signature) {
- var hmac = crypto.createHmac('sha1', config.SECRET_KEY);
- hmac.update(signature);
- var digest = hmac.digest('base64');
- return util.base64ToUrlsafe(digest);
-};
+// ------------------------------------------------------------------------------------------
+// type DownloadToken
-UploadToken.prototype.generateToken = function() {
- var signature = this.generateSignature();
- var encoded_digest = this.generateEncodedDigest(signature);
- return config.ACCESS_KEY + ":" + encoded_digest + ":" + signature;
+function DownloadToken(opts) {
+ this.expires = opts.expires || 3600;
+ this.pattern = opts.pattern || "*/*";
+}
+
+DownloadToken.prototype.generateToken = function() {
+ var params = {
+ S: this.pattern,
+ E: this.expires + Math.floor(Date.now() / 1000),
+ };
+ return generateToken(params);
};
-exports.UploadToken = UploadToken;
+// ------------------------------------------------------------------------------------------
View
148 lib/rs.js
@@ -1,6 +1,6 @@
var fs = require('fs');
var path = require('path');
-var crc = require('crc');
+var crc32 = require('crc32');
var mime = require('mime');
var formstream = require('formstream');
var config = require('./conf.js');
@@ -84,7 +84,7 @@ Service.prototype.upload = function(upToken, key, mimeType, filename, stream, on
**/
var self = this;
if (!mimeType) {
- mimeType = mime.lookup(filename);
+ mimeType = "application/octet-stream";
}
var entryURI = this.bucket + ':' + key;
entryURI = '/rs-put/' + util.encode(entryURI) + '/mimeType/' + util.encode(mimeType);
@@ -117,13 +117,17 @@ Service.prototype.uploadFile = function(upToken, key, mimeType, localFile, onret
});
};
-Service.prototype.uploadWithToken = function(uploadToken, localFile, stream, key, mimeType, customMeta, callbackParams, enableCrc32Check, onret) {
+Service.prototype.uploadWithToken = function(uploadToken, stream, key, mimeType, customMeta, callbackParams, crc32, onret) {
/*
- * func UploadWithToken(uploadToken, localFile, stream, key, mimeType, customMeta, callbackParams, enableCrc32Check, onret) => (data PutRet, code int, err Error)
+ * func UploadWithToken(uploadToken, stream, key, mimeType, customMeta, callbackParams, crc32, onret) => (data PutRet, code int, err Error)
* 使用upload_token以multipart/form-data形式上传ReadStream流
**/
var bucket = this.bucket;
- var actionString = util.generateActionString(localFile, bucket, key, mimeType, customMeta, enableCrc32Check);
+ if (!mimeType) {
+ mimeType = "application/octet-stream";
+ }
+
+ var actionString = util.generateActionString(bucket, key, mimeType, customMeta, crc32);
if (callbackParams === null) {
callbackParams = {
"bucket": bucket,
@@ -134,9 +138,8 @@ Service.prototype.uploadWithToken = function(uploadToken, localFile, stream, key
var callbackQueryString = util.generateQueryString(callbackParams);
var url = config.UP_HOST + "/upload";
- var filename = path.basename(localFile);
+ var filename = path.basename(key);
var form = formstream();
- var mimeType = mime.lookup(localFile);
form.field('action', actionString);
form.field('params', callbackQueryString);
form.field('multipart', true);
@@ -147,24 +150,37 @@ Service.prototype.uploadWithToken = function(uploadToken, localFile, stream, key
return this.conn.callWithToken(uploadToken, url, form, onret);
};
-
Service.prototype.uploadFileWithToken = function(uploadToken, localFile, key, mimeType, customMeta, callbackParams, enableCrc32Check, onret) {
/*
* func UploadFileWithToken(uploadToken, localFile, key, mimeType, customMeta, callbackParams, enableCrc32Check, onret) => (data PutRet, code int, err Error)
* 使用upload_token以multipart/form-data形式上传文件
**/
- var self = this;
+ var self = this
+ , bucket = self.bucket;
if (!mimeType) {
- mimeType = mime.lookup(localFile);
+ mimeType = mime.lookup(localFile);
+ }
+ fs.stat(localFile, function(err, fi) {
+ if (err) {
+ onret({code: -1, error: err.toString(), detail: err});
+ return;
}
- fs.stat(localFile, function(err, fi) {
- if (err) {
- onret({code: -1, error: err.toString(), detail: err});
- return;
- }
- var stream = fs.createReadStream(localFile);
- self.uploadWithToken(uploadToken, localFile, stream, key, mimeType, customMeta, callbackParams, enableCrc32Check, onret);
- });
+ var fileCrc32 = null
+ , stream = fs.createReadStream(localFile);
+
+ if (enableCrc32Check) {
+ var fileStat = fs.statSync(localFile)
+ , fileSize = fileStat.size
+ , buf = new Buffer(fileSize)
+ , fd = fs.openSync(localFile, 'r');
+
+ fs.readSync(fd, buf, 0, fileSize, 0);
+ fs.closeSync(fd);
+ fileCrc32 = parseInt("0x" + crc32(buf)).toString();
+ }
+
+ self.uploadWithToken(uploadToken, stream, key, mimeType, customMeta, callbackParams, fileCrc32, onret);
+ });
};
Service.prototype.get = function(key, attName, onret) {
@@ -234,6 +250,64 @@ Service.prototype.drop = function(onret) {
this.conn.callWith(url, null, onret);
};
+Service.prototype.copy = function(sourceBucket, sourceKey, targetBucket, targetKey, onret) {
+ /*
+ * func Copy(sourceBucket, sourceKey, targetBucket, targetKey, onret) => Bool
+ * 拷贝某个资源表中的文件到另一个资源表中的某个文件
+ */
+ var url = config.RS_HOST + generateMoveOrCopyOpString('copy', sourceBucket, sourceKey, targetBucket, targetKey);
+ this.conn.callWith(url, null, onret);
+};
+
+Service.prototype.move = function(sourceBucket, sourceKey, targetBucket, targetKey, onret) {
+ /*
+ * func Move(sourceBucket, sourceKey, targetBucket, targetKey, onret) => Bool
+ * 移动某个资源表中的文件到另一个资源表中的某个文件
+ */
+ var url = config.RS_HOST + generateMoveOrCopyOpString('move', sourceBucket, sourceKey, targetBucket, targetKey);
+ this.conn.callWith(url, null, onret);
+};
+
+Service.prototype.batchGet = function(bucket, keys, onret) {
+ /*
+ * func BatchGet(bucket, keys, onret) => GetRet[]
+ * 为每个key生成一个短期有效的下载地址
+ */
+ batch(this, "get", bucket, keys, onret);
+};
+
+Service.prototype.batchStat = function(bucket, keys, onret) {
+ /*
+ * func BatchStat(bucket, keys, onret) => Entry[]
+ * 查看每个key所对应文件的属性
+ */
+ batch(this, "stat", bucket, keys, onret);
+};
+
+Service.prototype.batchDelete = function(bucket, keys, onret) {
+ /*
+ * func BatchDelete(bucket, keys, onret) => Bool[]
+ * 批量删除每个key所对应的资源
+ */
+ batch(this, "delete", bucket, keys, onret);
+};
+
+Service.prototype.batchCopy = function(entries, onret) {
+ /*
+ * func BatchCopy(entries, onret) => Bool[]
+ * 批量拷贝文件
+ */
+ batchMoveOrCopy(this, 'copy', entries, onret);
+};
+
+Service.prototype.batchMove = function(entries, onret) {
+ /*
+ * func BatchMove(entries, onret) => Bool[]
+ * 批量移动文件
+ */
+ batchMoveOrCopy(this, 'move', entries, onret);
+};
+
/*
* 持久化存储一个经过云端服务处理过后的资源
*/
@@ -294,4 +368,42 @@ Service.prototype.unsetStyle = function(name, onret){
this.conn.callWith(url, null, onret);
};
+
+// ------------------------------------------------------------------------------------------
+// private functions
+
+function generateMoveOrCopyOpString(command, sourceBucket, sourceKey, targetBucket, targetKey) {
+ var sourceEntryURI = sourceBucket + ":" + sourceKey;
+ var targetEntryURI = targetBucket + ":" + targetKey;
+ var url = '/' + command + '/' + util.encode(sourceEntryURI) + '/' + util.encode(targetEntryURI);
+ return url;
+};
+
+function batch(rs, command, bucket, keys, onret) {
+ var ops = []
+ , length = keys.length
+ , url = config.RS_HOST + '/batch?';
+
+ for(var i = 0; i < length; i++) {
+ console.log("Entry URI is: ", bucket + ":" + keys[i]);
+ var encodedEntryURI = util.encode(bucket + ":" + keys[i]);
+ ops.push("op=/" + command + "/" + encodedEntryURI);
+ }
+ url += ops.join("&");
+ console.log("Batch URL: ", url);
+ rs.conn.callWith(url, null, onret);
+}
+
+function batchMoveOrCopy(rs, command, entries, onret) {
+ var ops = []
+ , length = ops.length
+ , url = config.RS_HOST + '/batch?';
+
+ for(var i = 0; i < length; i++) {
+ ops.push('op=' + moveOrCopy(command, entries[i][0], entries[i][1], entries[i][2], entries[i][3]));
+ }
+ url += ops.join("&");
+ rs.conn.callWith(url, null, onret);
+}
+
// ------------------------------------------------------------------------------------------
Oops, something went wrong.

0 comments on commit 0f0c840

Please sign in to comment.