feat(7.x): add crc64 webassembly crc64 c++ addon

.eslintcache

@@ -0,0 +1 @@
+[{"/Users/xutao/ali-oss/lib/common/utils/crc64/index.js":"1"},{"size":521,"mtime":1660813927865,"results":"2","hashOfConfig":"3"},{"filePath":"4","messages":"5","errorCount":0,"warningCount":4,"fixableErrorCount":0,"fixableWarningCount":0,"source":null},"t61pzs","/Users/xutao/ali-oss/lib/common/utils/crc64/index.js",["6","7","8","9"],{"ruleId":"10","severity":1,"message":"11","line":1,"column":33,"nodeType":"12","messageId":"13","endLine":1,"endColumn":51},{"ruleId":"14","severity":1,"message":"15","line":1,"column":33,"nodeType":"12","messageId":"16","endLine":1,"endColumn":51},{"ruleId":"10","severity":1,"message":"11","line":2,"column":15,"nodeType":"12","messageId":"13","endLine":2,"endColumn":39},{"ruleId":"14","severity":1,"message":"15","line":2,"column":15,"nodeType":"12","messageId":"16","endLine":2,"endColumn":39},"@typescript-eslint/no-var-requires","Require statement not part of import statement.","CallExpression","noVarReqs","@typescript-eslint/no-require-imports","A `require()` style import is forbidden.","noRequireImports"]

README.md

@@ -185,6 +185,10 @@ All operation use es7 async/await to implement. All api is async function.
   - [imgClient.listStyle([options])](#imgclientliststyleoptions)
   - [imgClient.deleteStyle(name[, options])](#imgclientdeletestylename-options)
   - [imgClient.signatureUrl(name)](#imgclientsignatureurlname)
+- [Utils](#utils)
+    - crc
+        - [.checkCrc64(content, result)](#checkcrc64)
+        - [.checkCrc64File(filePath,callback)](#checkcrc64file)
 - [Known Errors](#known-errors)
 
 ## Node Usage
@@ -1662,6 +1666,7 @@ parameters:
     - 'Expires' expires time for download, an absolute date and time. e.g.: `Tue, 08 Dec 2020 13:49:43 GMT`
     - See more: [PutObject](https://help.aliyun.com/document_detail/31978.html#title-yxe-96d-x61)
   - [disabledMD5] {Boolean} default true, it just work in Browser. if false,it means that MD5 is automatically calculated for uploaded files. **_NOTE:_** Synchronous computing tasks will block the main process
+  - [crc64] {Boolean} Whether to enable crc64. If you are using a browser environment please check the [oss-crc64-plug](https://github.com/taotao7/oss-crc64-plugin) plugin
 
 Success will return the object information.
 
@@ -1785,6 +1790,7 @@ parameters:
     - 'Content-Disposition' object name for download, e.g.: `Content-Disposition: somename`
     - 'Content-Encoding' object content encoding for download, e.g.: `Content-Encoding: gzip`
     - 'Expires' expires time for download, an absolute date and time. e.g.: `Tue, 08 Dec 2020 13:49:43 GMT`
+  - [crc64] {Boolean} Whether to enable crc64. If you are using a browser environment please check the [oss-crc64-plug](https://github.com/taotao7/oss-crc64-plugin) plugin
 
 Success will return the object information.
 
@@ -2020,6 +2026,7 @@ parameters:
         otherwise throw PreconditionFailedError
     - 'If-None-Match' object etag not equal this will return 200 and object meta,
         otherwise return 304 not modified
+  - [crc64] {Boolean} Whether to enable crc64. If you are using a browser environment please check the [oss-crc64-plug](https://github.com/taotao7/oss-crc64-plugin) plugin
 
 Success will return the info contains response.
 
@@ -3129,6 +3136,7 @@ parameters:
     - **NOTE**: Some headers are [disabled in browser][disabled-browser-headers]
   - [timeout] {Number} Milliseconds before a request is considered to be timed out
   - [disabledMD5] {Boolean} default true, it just work in Browser. if false,it means that MD5 is automatically calculated for uploaded files. **_NOTE:_** Synchronous computing tasks will block the main process
+  - [crc64] {Boolean} Whether to enable crc64. If you are using a browser environment please check the [oss-crc64-plug](https://github.com/taotao7/oss-crc64-plugin) plugin
 
 Success will return:
 
@@ -4367,6 +4375,42 @@ const url = imgClient.signatureUrl('
 // http://thumbnail.myimageservice.com/demo.jpg@200w_200h?OSSAccessKeyId=uZxyLARzYZtGwHKY&Expires=1427803849&Signature=JSPRe06%2FjQpQSj5zlx2ld1V%2B35I%3D
 ```
 
+## Utils
+
+### .checkCrc64
+Whether the calculation is consistent with the results
+If you are using a browser environment please check the [oss-crc64-plug](https://github.com/taotao7/oss-crc64-plugin) plugin
+
+parameters:
+
+- content {String|Buffer} what needs to be calculated
+- result  {String} results to be compared
+
+Success will return boolean
+
+```js
+store.checkCrc64(123456789, '11051210869376104954')
+store.checkCrc64(Buffer.from('123456789','11051210869376104954'))
+```
+
+### .checkCrc64File
+Compare Documents
+If you are using a browser environment please check the [oss-crc64-plug](https://github.com/taotao7/oss-crc64-plugin) plugin
+
+parameters:
+
+- filePath {String} file path
+- callback {Function} callback functions
+
+```js
+store.checkCrc64File(path.join(__dirname,'test.txt'),(err,res)=>{
+  if(!err){
+    // the returned crc64 value
+    console.log(res)
+  }
+})
+```
+
 ## Cluster Mode
 
 Cluster mode now only support object operations.

lib/browser/object/put.js

@@ -76,6 +76,12 @@ async function put(name, file, options = {}) {
     params.content = content;
     params.successStatuses = [200];
     const result = await this.request(params);
+    if (options.crc64) {
+        console.log('check crc64 put');
+        if (typeof options.crc64 === 'function' && options.crc64(content) !== result.res.headers['x-oss-hash-crc64ecma']) {
+            throw new Error('crc64 check fail');
+        }
+    }
     const ret = {
         name,
         url: objectUrl_1.objectUrl(name, this.options),

lib/common/multipart/handleUploadPart.js

@@ -5,7 +5,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.handleUploadPart = void 0;
 const copy_to_1 = __importDefault(require("copy-to"));
-const isStreamLike = (stream) => {
+const crc64_1 = require("../utils/crc64");
+const isStreamLike = stream => {
     return stream && stream.pipe;
 };
 /**
@@ -37,7 +38,32 @@ async function handleUploadPart(name, uploadId, partNo, data, options = {}) {
     }
     params.disabledMD5 = opt.disabledMD5;
     params.successStatuses = [200];
+    const isBrowserEnv = process && process.browser;
+    // current part buffer to calculator
+    let buffers = [];
+    if (opt.crc64) {
+        if (isBrowserEnv) {
+            buffers = params.content;
+        }
+        else {
+            params.stream.on('data', d => {
+                buffers.push(d);
+            });
+        }
+    }
     const result = await this.request(params);
+    // check crc64
+    if (options.crc64) {
+        if (isBrowserEnv) {
+            if (typeof options.crc64 === 'function' &&
+                options.crc64(buffers) !== result.res.headers['x-oss-hash-crc64ecma']) {
+                throw new Error('crc64 check fail');
+            }
+        }
+        else if (!crc64_1.checkCrc64(Buffer.concat(buffers), result.res.headers['x-oss-hash-crc64ecma'])) {
+            throw new Error('crc64 check fail');
+        }
+    }
     if (!result.res.headers.etag) {
         throw new Error('Please set the etag of expose-headers in OSS \n https://help.aliyun.com/document_detail/32069.html');
     }

lib/common/object/get.js

@@ -8,6 +8,7 @@ const fs_1 = __importDefault(require("fs"));
 const is_type_of_1 = __importDefault(require("is-type-of"));
 const deleteFileSafe_1 = require("../utils/deleteFileSafe");
 const isObject_1 = require("../utils/isObject");
+const crc64_1 = require("../utils/crc64");
 /**
  * get
  * @param {String} name - object name
@@ -44,6 +45,28 @@ async function get(name, file, options = {}) {
         params.writeStream = writeStream;
         params.successStatuses = [200, 206, 304];
         result = await this.request(params);
+        // buffer
+        if (options.crc64 && result.data) {
+            if (isBrowserEnv) {
+                if (typeof options.crc64 === 'function' &&
+                    options.crc64(result.data) !== result.headers['x-oss-hash-crc64ecma']) {
+                    throw new Error('crc64 check fail');
+                }
+            }
+            else if (!crc64_1.checkCrc64(result.data, result.headers['x-oss-hash-crc64ecma'])) {
+                throw new Error('crc64 check fail');
+            }
+        }
+        // stream
+        if (options.crc64 && writeStream) {
+            crc64_1.checkCrc64File(writeStream.path, (err, d) => {
+                if (err)
+                    throw new Error(err.toString());
+                if (d !== result.headers['x-oss-hash-crc64ecma']) {
+                    throw new Error('crc64 check fail');
+                }
+            });
+        }
         if (needDestroy) {
             writeStream.destroy();
         }

lib/common/utils/crc64/crc64.js

@@ -0,0 +1,53 @@
+const crc64 = require('crc64-ecma182');
+const _stream = require('stream');
+
+function check() {
+  let content;
+  let init_crc = 0;
+  const typeErrorMessage = 'Only (string|buffer) or (string|number, string|buffer) accepted!';
+  if (arguments.length === 1) {
+    content = arguments[0];
+  } else if (arguments.length === 2) {
+    content = arguments[0];
+    init_crc = Number(arguments[1]);
+  } else {
+    throw new TypeError(typeErrorMessage);
+  }
+
+  if (typeof content === 'string' || Buffer.isBuffer(content)) {
+    content = Buffer.from(content);
+  } else {
+    throw new TypeError(typeErrorMessage);
+  }
+
+  return crc64.crc64(content, init_crc);
+}
+
+function check_stream(stream, callback) {
+  if (!(stream instanceof _stream.Stream)) throw new TypeError('Only (stream, callback) accepted!');
+  let init_crc = 0;
+  stream.on('data', chunk => {
+    init_crc = check(chunk, init_crc);
+  });
+  stream.on('end', () => {
+    callback(null, init_crc);
+    try {
+      if (stream) stream.close();
+    } catch (e) {
+      console.log(e);
+    }
+  });
+  stream.on('error', err => {
+    callback(err, null);
+    try {
+      if (stream) stream.close();
+    } catch (e) {
+      console.log(e);
+    }
+  });
+}
+
+module.exports = {
+  check,
+  check_stream
+};

lib/common/utils/crc64/index.d.ts

@@ -0,0 +1,3 @@
+export declare const checkCrc64: (content: any, oss_crc64: any) => boolean;
+export declare const checkCrc64Stream: (p: any, callback: any) => any;
+export declare const checkCrc64File: (p: any, callback: any) => any;

lib/common/utils/crc64/index.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.checkCrc64File = exports.checkCrc64Stream = exports.checkCrc64 = void 0;
+// @ts-ignore
+const { check, check_stream } = require('./crc64.js');
+const CRC64 = require('crc64-ecma182');
+// @ts-ignore
+exports.checkCrc64 = (content, oss_crc64) => CRC64.toUInt64String(check(content)) === oss_crc64;
+exports.checkCrc64Stream = (p, callback) => check_stream(p, callback);
+exports.checkCrc64File = (p, callback) => CRC64.crc64File(p, true, callback);

lib/common/utils/index.d.ts

@@ -66,5 +66,7 @@ declare const _default: {
         _signatureForURL: typeof import("./signUtils")._signatureForURL;
     };
     WebFileReadStream: typeof WebFileReadStream;
+    checkCrc64: (content: any, oss_crc64: any) => boolean;
+    checkCrc64File: (p: any, callback: any) => any;
 };
 export default _default;

lib/common/utils/index.js

@@ -37,6 +37,7 @@ const parseXML_1 = require("./parseXML");
 const policy2Str_1 = require("./policy2Str");
 const signUtils_1 = __importDefault(require("./signUtils"));
 const webFileReadStream_1 = require("./webFileReadStream");
+const crc64_1 = require("./crc64");
 exports.default = {
     _getObjectMeta: _getObjectMeta_1._getObjectMeta,
     authorization: authorization_1.authorization,
@@ -73,5 +74,7 @@ exports.default = {
     parseXML: parseXML_1.parseXML,
     policy2Str: policy2Str_1.policy2Str,
     signUtils: signUtils_1.default,
-    WebFileReadStream: webFileReadStream_1.WebFileReadStream
+    WebFileReadStream: webFileReadStream_1.WebFileReadStream,
+    checkCrc64: crc64_1.checkCrc64,
+    checkCrc64File: crc64_1.checkCrc64File
 };

lib/node/object/put.js

@@ -15,6 +15,7 @@ const convertMetaToHeaders_1 = require("../../common/utils/convertMetaToHeaders"
 const objectUrl_1 = require("../../common/utils/objectUrl");
 const encodeCallback_1 = require("../../common/utils/encodeCallback");
 const isBuffer_1 = require("../../common/utils/isBuffer");
+const crc64_1 = require("../../common/utils/crc64");
 /**
  * put an object from String(file path)/Buffer/ReadableStream
  * @param {String} name the object key
@@ -63,6 +64,11 @@ async function put(name, file, options = {}) {
     params.content = content;
     params.successStatuses = [200];
     const result = await this.request(params);
+    if (options === null || options === void 0 ? void 0 : options.crc64) {
+        if (!crc64_1.checkCrc64(content, result.res.headers['x-oss-hash-crc64ecma'])) {
+            throw new Error('crc64 check faild');
+        }
+    }
     const ret = {
         name,
         url: objectUrl_1.objectUrl(name, this.options),

lib/node/object/putStream.js

@@ -10,6 +10,7 @@ const objectName_1 = require("../../common/utils/objectName");
 const convertMetaToHeaders_1 = require("../../common/utils/convertMetaToHeaders");
 const objectUrl_1 = require("../../common/utils/objectUrl");
 const encodeCallback_1 = require("../../common/utils/encodeCallback");
+const crc64_1 = require("../../common/utils/crc64");
 /**
  * put an object from ReadableStream. If `options.contentLength` is
  * not provided, chunked encoding is used.
@@ -41,6 +42,15 @@ async function putStream(name, stream, options = {}) {
     params.stream = pump_1.default(stream, transform);
     params.successStatuses = [200];
     const result = await this.request(params);
+    if (options.crc64) {
+        crc64_1.checkCrc64File(stream.path, (err, data) => {
+            if (err)
+                throw new Error(err.toString());
+            if (result.res.headers['x-oss-hash-crc64ecma'] !== data) {
+                throw new Error('crc64 check fail');
+            }
+        });
+    }
     const ret = {
         name,
         url: objectUrl_1.objectUrl(name, this.options),

lib/types/params.d.ts

@@ -21,6 +21,7 @@ export interface RequestOptions {
     subres?: Subres;
     ctx?: string;
     disabledMD5?: boolean;
+    crc64?: boolean | Function;
 }
 interface UserMeta {
     [propsName: string]: string;
@@ -51,11 +52,13 @@ export interface MultipartUploadOptions extends RequestOptions {
     callback?: ObjectCallback;
     copyheaders?: object;
     contentLength?: number;
+    crc64?: boolean | Function;
 }
 export interface GetObjectOptions extends MultiVersionCommonOptions {
     process?: string;
     /** only support Browser.js */
     responseCacheControl?: string;
+    crc64?: boolean | Function;
 }
 export interface PutObjectOptions extends RequestOptions {
     mime?: string;
@@ -64,6 +67,7 @@ export interface PutObjectOptions extends RequestOptions {
     contentLength?: number;
     method?: string;
     disabledMD5?: boolean;
+    crc64?: boolean | Function;
 }
 export interface PutBucketOptions extends RequestOptions {
     storageClass?: StorageType;

package.json

@@ -19,7 +19,8 @@
     "platform": "./shims/platform",
     "fs": false,
     "child_process": false,
-    "is-type-of": "./shims/is-type-of.js"
+    "is-type-of": "./shims/is-type-of.js",
+    "crc64-ecma182": false
   },
   "scripts": {
     "build-change-log": "standard-version",
@@ -129,6 +130,7 @@
     "agentkeepalive": "^3.4.1",
     "bowser": "^1.6.0",
     "copy-to": "^2.0.1",
+    "crc64-ecma182": "^1.0.0",
     "dateformat": "^2.0.0",
     "debug": "^2.2.0",
     "destroy": "^1.0.4",
@@ -154,4 +156,4 @@
       "npm run detect-secrets --"
     ]
   }
-}
+}

src/browser/object/put.ts

@@ -72,6 +72,12 @@ export async function put(this: any, name: string, file: any, options: PutObject
 
   const result = await this.request(params);
 
+  if (options.crc64) {
+    if (typeof options.crc64 === 'function' && options.crc64(content) !== result.res.headers['x-oss-hash-crc64ecma']) {
+      throw new Error('crc64 check fail');
+    }
+  }
+
   const ret: any = {
     name,
     url: objectUrl(name, this.options),

src/browser/object/putStream.ts

@@ -50,4 +50,3 @@ export async function putStream(this: any, name: string, stream: any, options: P
 
   return ret;
 }
-