Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: add single instance multi-multipartUpload interface warning #1162

Open
wants to merge 2 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
13 changes: 13 additions & 0 deletions .editorconfig
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
root = true

[*]
indent_style = space
indent_size = 2
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
quote_type = single

[*.md]
trim_trailing_whitespace = false
29 changes: 14 additions & 15 deletions .eslintignore
Original file line number Diff line number Diff line change
@@ -1,18 +1,17 @@
node_modules/
build/
coverage/
.tmp/
.git/
lib/browser/version.js
dist/
es/
lib/
node_modules/
**/*.min.js
**/*-min.js
**/*.bundle.js
example/
test/
browser-build.js
shims/
example/node_modules/
example/public/
example/public/index.js
test/.tmp/
test/browser/.tmp
test/browser/build
test/benchmark/
test/fixtures/
task/
browser-build.js
es
cjs
karma.conf.js
publish-check.js
publish.js
37 changes: 4 additions & 33 deletions .eslintrc.js
Original file line number Diff line number Diff line change
@@ -1,35 +1,6 @@
/* eslint max-len: [0] */
module.exports = {
extends: ['airbnb', 'eslint-config-ali/typescript'],
parserOptions: {
ecmaFeatures: {
experimentalObjectRestSpread: true,
},
},
env: {
browser: true,
node: true,
es6: true,
mocha: true,
jasmine: true,
jest: true,
},
rules: {
indent: ['error', 2],
// override default options
'no-underscore-dangle': [0],
'no-plusplus': [0],
'no-return-await':[0],
'no-param-reassign': [0],
'max-len': ['warn', 120, 2, {
ignoreUrls: true,
ignoreComments: false,
ignoreRegExpLiterals: true,
ignoreStrings: true,
ignoreTemplateLiterals: true,
}],
'no-buffer-constructor': [2],
"comma-dangle": [0],
'import/prefer-default-export': [0],
}
extends: [
'eslint-config-ali/node',
'prettier',
],
};
1 change: 1 addition & 0 deletions .husky/pre-commit
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@
. "$(dirname "$0")/_/husky.sh"

npm run lint-staged
npm run f2elint-scan
10 changes: 0 additions & 10 deletions .prettierrc

This file was deleted.

8 changes: 8 additions & 0 deletions .prettierrc.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
module.exports = {
printWidth: 100,
tabWidth: 2,
semi: true,
singleQuote: true,
trailingComma: 'all',
arrowParens: 'always',
};
4 changes: 3 additions & 1 deletion commitlint.config.js
Original file line number Diff line number Diff line change
@@ -1 +1,3 @@
module.exports = { extends: ['@commitlint/config-conventional'] };
module.exports = {
extends: ['ali'],
};
5 changes: 5 additions & 0 deletions f2elint.config.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
module.exports = {
enableStylelint: false,
enableMarkdownlint: false,
enablePrettier: true,
};
52 changes: 29 additions & 23 deletions lib/browser/managed-upload.js
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ const proto = exports;
* }
*/
proto.multipartUpload = async function multipartUpload(name, file, options = {}) {
if (this.options.multipartRunning) {
console.warn(
'multipartUpload is already executing, if you need to call more than one multipartUpload at the same time, please create more instances',
);
}
this.resetCancelFlag();
options.disabledMD5 = options.disabledMD5 === undefined ? true : !!options.disabledMD5;
if (options.checkpoint && options.checkpoint.uploadId) {
Expand Down Expand Up @@ -67,7 +72,7 @@ proto.multipartUpload = async function multipartUpload(name, file, options = {})
res: result.res,
bucket: this.options.bucket,
name,
etag: result.res.headers.etag
etag: result.res.headers.etag,
};

if ((options.headers && options.headers['x-oss-callback']) || options.callback) {
Expand All @@ -94,7 +99,7 @@ proto.multipartUpload = async function multipartUpload(name, file, options = {})
fileSize,
partSize,
uploadId,
doneParts: []
doneParts: [],
};

if (options && options.progress) {
Expand Down Expand Up @@ -136,14 +141,14 @@ proto._resumeMultipart = async function _resumeMultipart(checkpoint, options) {
const content = await self._createBuffer(file, pi.start, pi.end);
const data = {
content,
size: pi.end - pi.start
size: pi.end - pi.start,
};

let result;
try {
result = await self._uploadPart(name, uploadId, partNo, data, {
timeout: options.timeout,
disabledMD5: options.disabledMD5
disabledMD5: options.disabledMD5,
});
} catch (error) {
if (error.status === 404) {
Expand All @@ -154,7 +159,7 @@ proto._resumeMultipart = async function _resumeMultipart(checkpoint, options) {
if (!self.isCancel() && !multipartFinish) {
checkpoint.doneParts.push({
number: partNo,
etag: result.res.headers.etag
etag: result.res.headers.etag,
});

if (options.progress) {
Expand All @@ -163,7 +168,7 @@ proto._resumeMultipart = async function _resumeMultipart(checkpoint, options) {

resolve({
number: partNo,
etag: result.res.headers.etag
etag: result.res.headers.etag,
});
} else {
resolve();
Expand All @@ -184,31 +189,32 @@ proto._resumeMultipart = async function _resumeMultipart(checkpoint, options) {
};

const all = Array.from(new Array(numParts), (x, i) => i + 1);
const done = internalDoneParts.map(p => p.number);
const todo = all.filter(p => done.indexOf(p) < 0);
const done = internalDoneParts.map((p) => p.number);
const todo = all.filter((p) => done.indexOf(p) < 0);
const defaultParallel = 5;
const parallel = options.parallel || defaultParallel;

// upload in parallel
const jobErr = await this._parallel(
todo,
parallel,
value => new Promise((resolve, reject) => {
uploadPartJob(that, value)
.then(result => {
if (result) {
internalDoneParts.push(result);
}
resolve();
})
.catch(err => {
reject(err);
});
})
(value) =>
new Promise((resolve, reject) => {
uploadPartJob(that, value)
.then((result) => {
if (result) {
internalDoneParts.push(result);
}
resolve();
})
.catch((err) => {
reject(err);
});
}),
);
multipartFinish = true;

const abortEvent = jobErr.find(err => err.name === 'abort');
const abortEvent = jobErr.find((err) => err.name === 'abort');
if (abortEvent) throw abortEvent;

if (this.isCancel()) {
Expand Down Expand Up @@ -341,7 +347,7 @@ proto._getPartSize = function _getPartSize(fileSize, partSize) {
if (partSize < safeSize) {
partSize = safeSize;
console.warn(
`partSize has been set to ${partSize}, because the partSize you provided causes partNumber to be greater than 10,000`
`partSize has been set to ${partSize}, because the partSize you provided causes partNumber to be greater than 10,000`,
);
}
return partSize;
Expand All @@ -357,7 +363,7 @@ proto._divideParts = function _divideParts(fileSize, partSize) {

partOffs.push({
start,
end
end,
});
}

Expand Down
7 changes: 4 additions & 3 deletions lib/common/client/initOptions.js
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ module.exports = function (options) {
if (options.stsToken && !options.refreshSTSToken && !options.refreshSTSTokenInterval) {
console.warn(
"It's recommended to set 'refreshSTSToken' and 'refreshSTSTokenInterval' to refresh" +
' stsToken、accessKeyId、accessKeySecret automatically when sts token has expired'
' stsToken、accessKeyId、accessKeySecret automatically when sts token has expired',
);
}
if (options.bucket) {
Expand All @@ -46,9 +46,10 @@ module.exports = function (options) {
headerEncoding: 'utf-8',
refreshSTSToken: null,
refreshSTSTokenInterval: 60000 * 5,
retryMax: 0
retryMax: 0,
multipartRunning: false,
},
options
options,
);

opts.accessKeyId = opts.accessKeyId.trim();
Expand Down