diff --git a/examples/form_upload_simple.js b/examples/form_upload_simple.js index f2bb69d..4570269 100644 --- a/examples/form_upload_simple.js +++ b/examples/form_upload_simple.js @@ -1,37 +1,43 @@ -const qiniu = require('../index.js'); -const proc = require('process'); +const os = require('os'); -var bucket = proc.env.QINIU_TEST_BUCKET; -var accessKey = proc.env.QINIU_ACCESS_KEY; -var secretKey = proc.env.QINIU_SECRET_KEY; -var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); -var options = { +const qiniu = require('qiniu'); + +const bucket = process.env.QINIU_TEST_BUCKET; +const accessKey = process.env.QINIU_ACCESS_KEY; +const secretKey = process.env.QINIU_SECRET_KEY; +const mac = new qiniu.auth.digest.Mac(accessKey, secretKey); +const options = { scope: bucket }; -var putPolicy = new qiniu.rs.PutPolicy(options); +const putPolicy = new qiniu.rs.PutPolicy(options); -var uploadToken = putPolicy.uploadToken(mac); -var config = new qiniu.conf.Config(); -var localFile = '/Users/jemy/Downloads/download.csv'; +const uploadToken = putPolicy.uploadToken(mac); +const config = new qiniu.conf.Config(); +const localFile = os.homedir() + '/Downloads/83eda6926b94bb14.css'; // config.zone = qiniu.zone.Zone_z0; -var formUploader = new qiniu.form_up.FormUploader(config); -var putExtra = new qiniu.form_up.PutExtra(); +const formUploader = new qiniu.form_up.FormUploader(config); +const putExtra = new qiniu.form_up.PutExtra(); // file -putExtra.fname = 'test01.csv'; -putExtra.crc32 = 3497766758; -putExtra.metadata = { - 'x-qn-meta-name': 'qiniu' -}; -formUploader.putFile(uploadToken, null, localFile, putExtra, function (respErr, - respBody, respInfo) { - if (respErr) { - throw respErr; - } +// putExtra.fname = 'frontend-static-resource/widgets/_next/static/css/83eda6926b94bb14.css'; +// putExtra.metadata = { +// 'x-qn-meta-name': 'qiniu' +// }; +formUploader.putFile( + uploadToken, + 'frontend-static-resource/widgets/_next/static/css/83eda6926b94bb14.css', + localFile, + putExtra, + function (respErr, + respBody, respInfo) { + if (respErr) { + throw respErr; + } - if (respInfo.statusCode == 200) { - console.log(respBody); - } else { - console.log(respInfo.statusCode); - console.log(respBody); + if (respInfo.statusCode === 200) { + console.log(respBody); + } else { + console.log(respInfo.statusCode); + console.log(respBody); + } } -}); +); diff --git a/index.d.ts b/index.d.ts index 4b2fd71..2538925 100644 --- a/index.d.ts +++ b/index.d.ts @@ -3,7 +3,10 @@ * @date 2017-06-27 * @author xialeistudio */ -import { Callback } from 'urllib'; +import { Callback, RequestOptions } from 'urllib'; +import { Agent as HttpAgent, IncomingMessage} from 'http'; +import { Agent as HttpsAgent } from 'https'; +import { Readable } from "stream"; export declare type callback = (e?: Error, respBody?: any, respInfo?: any) => void; @@ -417,6 +420,150 @@ export declare namespace util { function isQiniuCallback(mac: auth.digest.Mac, requestURI: string, reqBody: string | null, callbackAuth: string): boolean; } +export declare namespace httpc { + interface ReqOpts { + agent?: HttpAgent; + httpsAgent?: HttpsAgent; + url: string; + middlewares: middleware.Middleware[]; + callback?: Callback; + urllibOptions: RequestOptions; + } + + interface RespWrapperOptions { + data: T; + resp: IncomingMessage; + } + + class RespWrapper { + data: T; + resp: IncomingMessage; + constructor(options: RespWrapperOptions); + ok(): boolean; + needRetry(): boolean; + } + + namespace middleware { + interface Middleware { + send( + request: ReqOpts, + next: (reqOpts: ReqOpts) => Promise> + ): Promise>; + } + + /** + * 组合中间件为一个调用函数 + * @param middlewares 中间件列表 + * @param handler 请求函数 + */ + function composeMiddlewares( + middlewares: Middleware[], + handler: (reqOpts: ReqOpts) => Promise> + ); + + /** + * 设置 User-Agent 请求头中间件 + */ + class UserAgentMiddleware implements Middleware { + constructor(sdkVersion: string); + send( + request: httpc.ReqOpts, + next: (reqOpts: httpc.ReqOpts) => Promise> + ): Promise>; + } + + interface RetryDomainsMiddlewareOptions { + backupDomains: string[]; + maxRetryTimes: number; + retryCondition: () => boolean; + } + + class RetryDomainsMiddleware implements Middleware { + /** + * 备用域名 + */ + backupDomains: string[]; + + /** + * 最大重试次数，包括首次请求 + */ + maxRetryTimes: number; + + /** + * 是否可以重试，可以通过该函数配置更详细的重试规则 + */ + retryCondition: () => boolean; + + /** + * 已经重试的次数 + * @private + */ + private _retriedTimes: number; + + /** + * 实例化重试域名中间件 + * @param retryDomainsOptions + */ + constructor(retryDomainsOptions: RetryDomainsMiddlewareOptions) + + /** + * 重试域名中间件逻辑 + * @param request + * @param next + */ + send( + request: httpc.ReqOpts, + next: (reqOpts: httpc.ReqOpts) => Promise> + ): Promise>; + + /** + * 控制重试逻辑，主要为 {@link retryCondition} 服务。若没有设置 retryCondition，默认 2xx 才会终止重试 + * @param err + * @param respWrapper + * @param reqOpts + * @private + */ + private _shouldRetry( + err: Error | null, + respWrapper: RespWrapper, + reqOpts: ReqOpts + ): boolean; + } + } + + interface HttpClientOptions { + httpAgent?: HttpAgent; + httpsAgent?: HttpsAgent; + middlewares?: middleware.Middleware[]; + } + + interface GetOptions extends ReqOpts { + params: Record; + headers: Record; + } + + interface PostOptions extends ReqOpts { + data: string | Buffer | Readable; + headers: Record; + } + + interface PutOptions extends ReqOpts { + data: string | Buffer | Readable; + headers: Record + } + + class HttpClient { + httpAgent: HttpAgent; + httpsAgent: HttpsAgent; + middlewares: middleware.Middleware[]; + constructor(options: HttpClientOptions) + sendRequest(requestOptions: ReqOpts): Promise + get(getOptions: GetOptions): Promise + post(postOptions: PostOptions): Promise + put(putOptions: PutOptions): Promise + } +} + export declare namespace rpc { type Headers = Record & { 'User-Agent'?: string; @@ -428,6 +575,8 @@ export declare namespace rpc { mac: auth.digest.Mac; } + const qnHttpClient: httpc.HttpClient; + /** * * @param requestUrl 请求地址 diff --git a/index.js b/index.js index f65a218..58d74d1 100644 --- a/index.js +++ b/index.js @@ -8,6 +8,11 @@ module.exports = { rs: require('./qiniu/storage/rs.js'), fop: require('./qiniu/fop.js'), conf: require('./qiniu/conf.js'), + httpc: { + middleware: require('./qiniu/httpc/middleware'), + HttpClient: require('./qiniu/httpc/client').HttpClient, + ResponseWrapper: require('./qiniu/httpc/responseWrapper').ResponseWrapper + }, rpc: require('./qiniu/rpc.js'), util: require('./qiniu/util.js'), zone: require('./qiniu/zone.js'), diff --git a/qiniu/conf.js b/qiniu/conf.js index f40af5f..3c318ca 100644 --- a/qiniu/conf.js +++ b/qiniu/conf.js @@ -18,13 +18,30 @@ exports.FormMimeJson = 'application/json'; exports.FormMimeRaw = 'application/octet-stream'; exports.RS_HOST = 'rs.qiniu.com'; exports.RPC_TIMEOUT = 600000; // 600s -exports.UC_HOST = 'uc.qbox.me'; +let UC_BACKUP_HOSTS = [ + 'kodo-config.qiniuapi.com', + 'api.qiniu.com' +]; +Object.defineProperty(exports, 'UC_BACKUP_HOSTS', { + get: () => UC_BACKUP_HOSTS, + set: v => { + UC_BACKUP_HOSTS = v; + } +}); +let UC_HOST = 'uc.qbox.me'; +Object.defineProperty(exports, 'UC_HOST', { + get: () => UC_HOST, + set: v => { + UC_HOST = v; + UC_BACKUP_HOSTS = []; + } +}); // proxy exports.RPC_HTTP_AGENT = null; exports.RPC_HTTPS_AGENT = null; -exports.Config = function Config(options) { +exports.Config = function Config (options) { options = options || {}; // use http or https protocol this.useHttpsDomain = !!(options.useHttpsDomain || false); diff --git a/qiniu/httpc/client.js b/qiniu/httpc/client.js new file mode 100644 index 0000000..bdadfe0 --- /dev/null +++ b/qiniu/httpc/client.js @@ -0,0 +1,241 @@ +const http = require('http'); +const https = require('https'); + +const urllib = require('urllib'); + +const middleware = require('./middleware'); +const { ResponseWrapper } = require('./responseWrapper'); + +/** + * + * @param {Object} options + * @param {http.Agent} [options.httpAgent] + * @param {https.Agent} [options.httpsAgent] + * @param {middleware.Middleware[]} [options.middlewares] + * + * @constructor + */ +function HttpClient (options) { + this.httpAgent = options.httpAgent || http.globalAgent; + this.httpsAgent = options.httpsAgent || https.globalAgent; + this.middlewares = options.middlewares || []; +} + +HttpClient.prototype._handleRequest = function (req) { + return new Promise((resolve, reject) => { + try { + urllib.request(req.url, req.urllibOptions, (err, data, resp) => { + if (err) { + err.resp = resp; + reject(err); + return; + } + resolve(new ResponseWrapper({ data, resp })); + }); + } catch (e) { + reject(e); + } + }); +}; + +/** + * Options for request + * @typedef {Object} ReqOpts + * @property {http.Agent} [agent] + * @property {https.Agent} [httpsAgent] + * @property {string} url + * @property {middleware.Middleware[]} middlewares + * @property {urllib.Callback} callback + * @property {urllib.RequestOptions} urllibOptions + */ + +/** + * Wrapped result of request + * @typedef {Object} RespWrapper + * @property {*} data + * @property {http.IncomingMessage} resp + */ + +/** + * + * @param {ReqOpts} requestOptions + * @return {Promise} + */ +HttpClient.prototype.sendRequest = function (requestOptions) { + const mwList = this.middlewares.concat(requestOptions.middlewares); + + if (!requestOptions.agent) { + requestOptions.agent = this.httpAgent; + } + + if (!requestOptions.httpsAgent) { + requestOptions.httpsAgent = this.httpsAgent; + } + + const handle = middleware.composeMiddlewares( + mwList, + this._handleRequest + ); + + const resPromise = handle(requestOptions); + + if (requestOptions.callback) { + resPromise + .then(({ data, resp }) => + requestOptions.callback(null, data, resp) + ) + .catch(err => { + requestOptions.callback(err, null, err.resp); + }); + } + + return resPromise; +}; + +/** + * @param {Object} reqOptions + * @param {string} reqOptions.url + * @param {http.Agent} [reqOptions.agent] + * @param {https.Agent} [reqOptions.httpsAgent] + * @param {Object} [reqOptions.params] + * @param {Object} [reqOptions.headers] + * @param {middleware.Middleware[]} [reqOptions.middlewares] + * @param {urllib.RequestOptions} [urllibOptions] + * @return {Promise} + */ +HttpClient.prototype.get = function (reqOptions, urllibOptions) { + const { + url, + params, + headers, + middlewares, + agent, + httpsAgent, + callback + } = reqOptions; + + urllibOptions = urllibOptions || {}; + urllibOptions.method = 'GET'; + urllibOptions.headers = Object.assign( + { + Connection: 'keep-alive' + }, + headers, + urllibOptions.headers || {} + ); + urllibOptions.data = params; + urllibOptions.followRedirect = true; + + return this.sendRequest({ + url: url, + middlewares: middlewares || [], + agent: agent, + httpsAgent: httpsAgent, + callback: callback, + urllibOptions: urllibOptions + }); +}; + +/** + * @param {Object} reqOptions + * @param {string} reqOptions.url + * @param {http.Agent} [reqOptions.agent] + * @param {https.Agent} [reqOptions.httpsAgent] + * @param {string | Buffer | Readable} [reqOptions.data] + * @param {Object} [reqOptions.headers] + * @param {middleware.Middleware[]} [reqOptions.middlewares] + * @param {urllib.RequestOptions} [urllibOptions] + * @return {Promise} + */ +HttpClient.prototype.post = function (reqOptions, urllibOptions) { + const { + url, + data, + headers, + middlewares, + agent, + httpsAgent, + callback + } = reqOptions; + + urllibOptions = urllibOptions || {}; + urllibOptions.method = 'POST'; + urllibOptions.headers = Object.assign( + { + Connection: 'keep-alive' + }, + headers, + urllibOptions.headers || {} + ); + urllibOptions.gzip = true; + + if (Buffer.isBuffer(data) || typeof data === 'string') { + urllibOptions.content = data; + } else if (data) { + urllibOptions.stream = data; + } else { + urllibOptions.headers['Content-Length'] = '0'; + } + + return this.sendRequest({ + url: url, + middlewares: middlewares || [], + agent: agent, + httpsAgent: httpsAgent, + callback: callback, + urllibOptions: urllibOptions + }); +}; + +/** + * @param {Object} reqOptions + * @param {string} reqOptions.url + * @param {http.Agent} [reqOptions.agent] + * @param {https.Agent} [reqOptions.httpsAgent] + * @param {string | Buffer | ReadableStream} [reqOptions.data] + * @param {Object} [reqOptions.headers] + * @param {middleware.Middleware[]} [reqOptions.middlewares] + * @param {urllib.RequestOptions} [urllibOptions] + * @return {Promise} + */ +HttpClient.prototype.put = function (reqOptions, urllibOptions) { + const { + url, + data, + headers, + middlewares, + agent, + httpsAgent, + callback + } = reqOptions; + + urllibOptions = urllibOptions || {}; + urllibOptions.method = 'PUT'; + urllibOptions.headers = Object.assign( + { + Connection: 'keep-alive' + }, + headers, + urllibOptions.headers || {} + ); + urllibOptions.gzip = true; + + if (Buffer.isBuffer(data) || typeof data === 'string') { + urllibOptions.content = data; + } else if (data) { + urllibOptions.stream = data; + } else { + urllibOptions.headers['Content-Length'] = '0'; + } + + return this.sendRequest({ + url: url, + middlewares: middlewares || [], + agent: agent, + httpsAgent: httpsAgent, + callback: callback, + urllibOptions: urllibOptions + }); +}; + +exports.HttpClient = HttpClient; diff --git a/qiniu/httpc/middleware/base.js b/qiniu/httpc/middleware/base.js new file mode 100644 index 0000000..2f3c4b3 --- /dev/null +++ b/qiniu/httpc/middleware/base.js @@ -0,0 +1,31 @@ +/** + * Middleware could be an interface if migrate to typescript + * @class + * @constructor + */ +function Middleware () { +} + +/** + * @memberOf Middleware + * @param _request + * @param _next + */ +Middleware.prototype.send = function (_request, _next) { + throw new Error('The Middleware NOT be Implemented'); +}; + +exports.Middleware = Middleware; + +/** + * @param {Middleware[]} middlewares + * @param {function(ReqOpts):Promise} handler + * @return {function(ReqOpts):Promise} + */ +exports.composeMiddlewares = function (middlewares, handler) { + return middlewares.reverse() + .reduce( + (h, mw) => request => mw.send(request, h), + handler + ); +}; diff --git a/qiniu/httpc/middleware/index.js b/qiniu/httpc/middleware/index.js new file mode 100644 index 0000000..97c550f --- /dev/null +++ b/qiniu/httpc/middleware/index.js @@ -0,0 +1,8 @@ +const base = require('./base'); + +module.exports = { + composeMiddlewares: base.composeMiddlewares, + Middleware: base.Middleware, + RetryDomainsMiddleware: require('./retryDomains').RetryDomainsMiddleware, + UserAgentMiddleware: require('./ua').UserAgentMiddleware +}; diff --git a/qiniu/httpc/middleware/retryDomains.js b/qiniu/httpc/middleware/retryDomains.js new file mode 100644 index 0000000..597e7ae --- /dev/null +++ b/qiniu/httpc/middleware/retryDomains.js @@ -0,0 +1,96 @@ +const middleware = require('./base'); + +/** + * @class + * @extends middleware.Middleware + * @param {Object} retryDomainsOptions + * @param {string[]} retryDomainsOptions.backupDomains + * @param {number} [retryDomainsOptions.maxRetryTimes] + * @param {function(Error || null, RespWrapper || null, ReqOpts):boolean} [retryDomainsOptions.retryCondition] + * @constructor + */ +function RetryDomainsMiddleware (retryDomainsOptions) { + this.backupDomains = retryDomainsOptions.backupDomains; + this.maxRetryTimes = retryDomainsOptions.maxRetryTimes || 2; + this.retryCondition = retryDomainsOptions.retryCondition; + + this._retriedTimes = 0; +} + +RetryDomainsMiddleware.prototype = Object.create(middleware.Middleware.prototype); +RetryDomainsMiddleware.prototype.constructor = RetryDomainsMiddleware; + +/** + * @memberOf RetryDomainsMiddleware + * @param {Error || null} err + * @param {RespWrapper || null} respWrapper + * @param {ReqOpts} reqOpts + * @return {boolean} + * @private + */ +RetryDomainsMiddleware.prototype._shouldRetry = function (err, respWrapper, reqOpts) { + if (typeof this.retryCondition === 'function') { + return this.retryCondition(err, respWrapper, reqOpts); + } + + return !respWrapper || respWrapper.needRetry(); +}; + +/** + * @memberOf RetryDomainsMiddleware + * @param {ReqOpts} reqOpts + * @param {function(ReqOpts):Promise} next + * @return {Promise} + */ +RetryDomainsMiddleware.prototype.send = function (reqOpts, next) { + const url = new URL(reqOpts.url); + const domains = this.backupDomains.slice(); // copy for late pop + + const couldRetry = () => { + // the reason `this.maxRetryTimes - 1` is request send first then add retriedTimes + // and `this.maxRetryTimes` means max request times per domain + if (this._retriedTimes < this.maxRetryTimes - 1) { + this._retriedTimes += 1; + return true; + } + + if (domains.length) { + this._retriedTimes = 0; + url.hostname = domains.shift(); + reqOpts.url = url.toString(); + return true; + } + + return false; + }; + + const tryNext = () => { + return next(reqOpts) + .then(respWrapper => { + if (!this._shouldRetry(null, respWrapper, reqOpts)) { + return respWrapper; + } + + if (couldRetry()) { + return tryNext(); + } + + return respWrapper; + }) + .catch(err => { + if (!this._shouldRetry(err, null, reqOpts)) { + return Promise.reject(err); + } + + if (couldRetry()) { + return tryNext(); + } + + return Promise.reject(err); + }); + }; + + return tryNext(); +}; + +exports.RetryDomainsMiddleware = RetryDomainsMiddleware; diff --git a/qiniu/httpc/middleware/ua.js b/qiniu/httpc/middleware/ua.js new file mode 100644 index 0000000..974ea55 --- /dev/null +++ b/qiniu/httpc/middleware/ua.js @@ -0,0 +1,36 @@ +const os = require('os'); + +const middleware = require('./base'); + +/** + * @class + * @extends middleware.Middleware + * @param {string} sdkVersion + * @constructor + */ +function UserAgentMiddleware (sdkVersion) { + this.userAgent = 'QiniuNodejs/' + sdkVersion + + ' (' + + os.type() + '; ' + + os.platform() + '; ' + + os.arch() + '; ' + + 'Node.js ' + process.version + '; )'; +} +UserAgentMiddleware.prototype = Object.create(middleware.Middleware.prototype); +UserAgentMiddleware.prototype.constructor = UserAgentMiddleware; + +/** + * @memberOf UserAgentMiddleware + * @param {ReqOpts} reqOpts + * @param {function(ReqOpts):Promise} next + * @return {Promise} + */ +UserAgentMiddleware.prototype.send = function (reqOpts, next) { + if (!reqOpts.urllibOptions.headers) { + reqOpts.urllibOptions.headers = {}; + } + reqOpts.urllibOptions.headers['User-Agent'] = this.userAgent; + return next(reqOpts); +}; + +exports.UserAgentMiddleware = UserAgentMiddleware; diff --git a/qiniu/httpc/responseWrapper.js b/qiniu/httpc/responseWrapper.js new file mode 100644 index 0000000..75f5095 --- /dev/null +++ b/qiniu/httpc/responseWrapper.js @@ -0,0 +1,34 @@ +function ResponseWrapper ({ + data, + resp +}) { + this.data = data; + this.resp = resp; +} + +/** + * @return {boolean} + */ +ResponseWrapper.prototype.ok = function () { + return this.resp && Math.floor(this.resp.statusCode / 100) === 2; +}; + +/** + * @return {boolean} + */ +ResponseWrapper.prototype.needRetry = function () { + if (this.resp.statusCode > 0 && this.resp.statusCode < 500) { + return false; + } + + // https://developer.qiniu.com/fusion/kb/1352/the-http-request-return-a-status-code + if ([ + 501, 509, 573, 579, 608, 612, 614, 616, 618, 630, 631, 632, 640, 701 + ].includes(this.resp.statusCode)) { + return false; + } + + return true; +}; + +exports.ResponseWrapper = ResponseWrapper; diff --git a/qiniu/rpc.js b/qiniu/rpc.js index 0d86932..6395f9a 100644 --- a/qiniu/rpc.js +++ b/qiniu/rpc.js @@ -1,8 +1,21 @@ -var urllib = require('urllib'); -var conf = require('./conf'); +const pkg = require('../package.json'); +const conf = require('./conf'); const digest = require('./auth/digest'); const util = require('./util'); - +const client = require('./httpc/client'); +const middleware = require('./httpc/middleware'); + +let uaMiddleware = new middleware.UserAgentMiddleware(pkg.version); +uaMiddleware = Object.defineProperty(uaMiddleware, 'userAgent', { + get: function () { + return conf.USER_AGENT; + } +}); +exports.qnHttpClient = new client.HttpClient({ + middlewares: [ + uaMiddleware + ] +}); exports.get = get; exports.post = post; exports.put = put; @@ -93,12 +106,12 @@ function postWithOptions (requestURI, requestForm, options, callbackFunc) { return post(requestURI, requestForm, headers, callbackFunc); } -function postMultipart(requestURI, requestForm, callbackFunc) { +function postMultipart (requestURI, requestForm, callbackFunc) { return post(requestURI, requestForm, requestForm.headers(), callbackFunc); } -function postWithForm(requestURI, requestForm, token, callbackFunc) { - var headers = { +function postWithForm (requestURI, requestForm, token, callbackFunc) { + const headers = { 'Content-Type': 'application/x-www-form-urlencoded' }; if (token) { @@ -107,8 +120,8 @@ function postWithForm(requestURI, requestForm, token, callbackFunc) { return post(requestURI, requestForm, headers, callbackFunc); } -function postWithoutForm(requestURI, token, callbackFunc) { - var headers = { +function postWithoutForm (requestURI, token, callbackFunc) { + const headers = { 'Content-Type': 'application/x-www-form-urlencoded' }; if (token) { @@ -118,16 +131,9 @@ function postWithoutForm(requestURI, token, callbackFunc) { } function get (requestUrl, headers, callbackFunc) { - headers = headers || {}; - headers['User-Agent'] = headers['User-Agent'] || conf.USER_AGENT; - headers.Connection = 'keep-alive'; - const data = { - method: 'GET', - headers: headers, dataType: 'json', - timeout: conf.RPC_TIMEOUT, - gzip: true + timeout: conf.RPC_TIMEOUT }; if (conf.RPC_HTTP_AGENT) { @@ -138,22 +144,16 @@ function get (requestUrl, headers, callbackFunc) { data.httpsAgent = conf.RPC_HTTPS_AGENT; } - return urllib.request( - requestUrl, - data, - callbackFunc - ); + return exports.qnHttpClient.get({ + url: requestUrl, + headers: headers, + callback: callbackFunc + }, data); } -function post(requestURI, requestForm, headers, callbackFunc) { +function post (requestURL, requestForm, headers, callbackFunc) { // var start = parseInt(Date.now() / 1000); - headers = headers || {}; - headers['User-Agent'] = headers['User-Agent'] || conf.USER_AGENT; - headers.Connection = 'keep-alive'; - - var data = { - headers: headers, - method: 'POST', + const data = { dataType: 'json', timeout: conf.RPC_TIMEOUT, gzip: true @@ -168,31 +168,17 @@ function post(requestURI, requestForm, headers, callbackFunc) { data.httpsAgent = conf.RPC_HTTPS_AGENT; } - if (Buffer.isBuffer(requestForm) || typeof requestForm === 'string') { - data.content = requestForm; - } else if (requestForm) { - data.stream = requestForm; - } else { - data.headers['Content-Length'] = 0; - } - - var req = urllib.request(requestURI, data, function (respErr, respBody, - respInfo) { - callbackFunc(respErr, respBody, respInfo); - }); - - return req; + return exports.qnHttpClient.post({ + url: requestURL, + data: requestForm, + headers: headers, + callback: callbackFunc + }, data); } -function put(requestURL, requestForm, headers, callbackFunc) { +function put (requestURL, requestForm, headers, callbackFunc) { // var start = parseInt(Date.now() / 1000); - headers = headers || {}; - headers['User-Agent'] = headers['User-Agent'] || conf.USER_AGENT; - headers.Connection = 'keep-alive'; - - var data = { - headers: headers, - method: 'PUT', + const data = { dataType: 'json', timeout: conf.RPC_TIMEOUT, gzip: true @@ -207,17 +193,10 @@ function put(requestURL, requestForm, headers, callbackFunc) { data.httpsAgent = conf.RPC_HTTPS_AGENT; } - if (Buffer.isBuffer(requestForm) || typeof requestForm === 'string') { - data.content = requestForm; - } else if (requestForm) { - data.stream = requestForm; - } else { - data.headers['Content-Length'] = 0; - } - - var req = urllib.request(requestURL, data, function (err, ret, info) { - callbackFunc(err, ret, info); - }); - - return req; + return exports.qnHttpClient.put({ + url: requestURL, + data: requestForm, + headers: headers, + callback: callbackFunc + }, data); } diff --git a/qiniu/zone.js b/qiniu/zone.js index e73102c..2c08fd2 100644 --- a/qiniu/zone.js +++ b/qiniu/zone.js @@ -1,6 +1,6 @@ -const urllib = require('urllib'); -const util = require('util'); const conf = require('./conf'); +const { RetryDomainsMiddleware } = require('./httpc/middleware'); +const rpc = require('./rpc'); // huadong exports.Zone_z0 = new conf.Zone([ @@ -73,65 +73,73 @@ exports.Zone_ap_northeast_1 = new conf.Zone([ 'api-ap-northeast-1.qiniuapi.com'); exports.getZoneInfo = function (accessKey, bucket, callbackFunc) { - const apiAddr = util.format( - 'https://%s/v4/query?ak=%s&bucket=%s', - conf.UC_HOST, - accessKey, - bucket - ); - urllib.request(apiAddr, function (respErr, respData, respInfo) { - if (respErr) { - callbackFunc(respErr, null, null); - return; - } + const apiAddr = 'https://' + conf.UC_HOST + '/v4/query'; - if (respInfo.statusCode != 200) { - // not ok - respErr = new Error(respInfo.statusCode + '\n' + respData); - callbackFunc(respErr, null, null); - return; - } + rpc.qnHttpClient.get({ + url: apiAddr, + params: { + ak: accessKey, + bucket: bucket + }, + middlewares: [ + new RetryDomainsMiddleware({ + backupDomains: conf.UC_BACKUP_HOSTS + }) + ], + callback: function (respErr, respData, respInfo) { + if (respErr) { + callbackFunc(respErr, null, null); + return; + } - let zoneData; - try { - const hosts = JSON.parse(respData).hosts; - if (!hosts || !hosts.length) { - respErr = new Error('no host available: ' + respData); + if (respInfo.statusCode !== 200) { + // not ok + respErr = new Error(respInfo.statusCode + '\n' + respData); callbackFunc(respErr, null, null); return; } - zoneData = hosts[0]; - } catch (err) { - callbackFunc(err, null, null); - return; - } - let srcUpHosts = []; - let cdnUpHosts = []; - let zoneExpire = 0; - try { - zoneExpire = zoneData.ttl; - // read src hosts - srcUpHosts = zoneData.up.domains; + let zoneData; + try { + const hosts = JSON.parse(respData).hosts; + if (!hosts || !hosts.length) { + respErr = new Error('no host available: ' + respData); + callbackFunc(respErr, null, null); + return; + } + zoneData = hosts[0]; + } catch (err) { + callbackFunc(err, null, null); + return; + } + let srcUpHosts = []; + let cdnUpHosts = []; + let zoneExpire = 0; - // read acc hosts - cdnUpHosts = zoneData.up.domains; + try { + zoneExpire = zoneData.ttl; + // read src hosts + srcUpHosts = zoneData.up.domains; - const ioHost = zoneData.io.domains[0]; - const rsHost = zoneData.rs.domains[0]; - const rsfHost = zoneData.rsf.domains[0]; - const apiHost = zoneData.api.domains[0]; - const zoneInfo = new conf.Zone( - srcUpHosts, - cdnUpHosts, - ioHost, - rsHost, - rsfHost, - apiHost - ); - callbackFunc(null, zoneInfo, zoneExpire); - } catch (e) { - callbackFunc(e, null, null); + // read acc hosts + cdnUpHosts = zoneData.up.domains; + + const ioHost = zoneData.io.domains[0]; + const rsHost = zoneData.rs.domains[0]; + const rsfHost = zoneData.rsf.domains[0]; + const apiHost = zoneData.api.domains[0]; + const zoneInfo = new conf.Zone( + srcUpHosts, + cdnUpHosts, + ioHost, + rsHost, + rsfHost, + apiHost + ); + callbackFunc(null, zoneInfo, zoneExpire); + } catch (e) { + callbackFunc(e, null, null); + } } }); }; diff --git a/test/httpc.test.js b/test/httpc.test.js new file mode 100644 index 0000000..4e97dac --- /dev/null +++ b/test/httpc.test.js @@ -0,0 +1,189 @@ +const should = require('should'); + +const qiniu = require('../index'); + +const { + Middleware, + RetryDomainsMiddleware +} = qiniu.httpc.middleware; + +describe('test http module', function () { + describe('test http ResponseWrapper', function () { + const { ResponseWrapper } = qiniu.httpc; + + it('needRetry', function () { + const cases = Array.from({ + length: 800 + }, (_, i) => { + if (i > 0 && i < 500) { + return { + code: i, + shouldRetry: false + }; + } + if ([ + 501, 509, 573, 579, 608, 612, 614, 616, 618, 630, 631, 632, 640, 701 + ].includes(i)) { + return { + code: i, + shouldRetry: false + }; + } + return { + code: i, + shouldRetry: true + }; + }); + cases.unshift({ + code: -1, + shouldRetry: true + }); + + const mockedResponseWrapper = new ResponseWrapper({ + data: [], + resp: { + statusCode: 200 + } + }); + + for (const item of cases) { + mockedResponseWrapper.resp.statusCode = item.code; + mockedResponseWrapper.needRetry().should.eql( + item.shouldRetry, + `${item.code} need${item.shouldRetry ? '' : ' NOT'} retry` + ); + } + }); + }); + + class OrderRecordMiddleware extends Middleware { + /** + * + * @param {string[]} record + * @param {string} label + */ + constructor (record, label) { + super(); + this.record = record; + this.label = label; + } + + /** + * + * @param {ReqOpts} request + * @param {function(ReqOpts):Promise} next + * @return {Promise} + */ + send (request, next) { + this.record.push(`bef_${this.label}${this.record.length}`); + return next(request).then((respWrapper) => { + this.record.push(`aft_${this.label}${this.record.length}`); + return respWrapper; + }); + } + } + + describe('test http middleware', function () { + it('test middleware', function (done) { + const recordList = []; + qiniu.rpc.qnHttpClient.sendRequest({ + url: 'https://qiniu.com/index.html', + urllibOptions: { + method: 'GET', + followRedirect: true + }, + middlewares: [ + new OrderRecordMiddleware(recordList, 'A'), + new OrderRecordMiddleware(recordList, 'B') + ] + }) + .then(({ + _data, + resp + }) => { + recordList.should.eql(['bef_A0', 'bef_B1', 'aft_B2', 'aft_A3']); + should.equal(resp.statusCode, 200); + done(); + }) + .catch(err => { + done(err); + }); + }); + + it('test retry domains', function (done) { + const recordList = []; + qiniu.rpc.qnHttpClient.sendRequest({ + url: 'https://fake.nodesdk.qiniu.com/index.html', + // url: 'https://qiniu.com/index.html', + urllibOptions: { + method: 'GET', + followRedirect: true + }, + middlewares: [ + new RetryDomainsMiddleware({ + backupDomains: [ + 'unavailable.pysdk.qiniu.com', + 'qiniu.com' + ], + maxRetryTimes: 3 + }), + new OrderRecordMiddleware(recordList, 'A') + ] + }) + .then(({ + _data, + _resp + }) => { + recordList.should.eql([ + // fake.nodesdk.qiniu.com + 'bef_A0', + 'bef_A1', + 'bef_A2', + // unavailable.pysdk.qiniu.com + 'bef_A3', + 'bef_A4', + 'bef_A5', + // qiniu.com + 'bef_A6', + 'aft_A7' + ]); + done(); + }) + .catch(err => { + done(err); + }); + }); + + it('test retry domains fail fast', function (done) { + const recordList = []; + qiniu.rpc.qnHttpClient.sendRequest({ + url: 'https://fake.nodesdk.qiniu.com/index.html', + // url: 'https://qiniu.com/index.html', + urllibOptions: { + method: 'GET', + followRedirect: true + }, + middlewares: [ + new RetryDomainsMiddleware({ + backupDomains: [ + 'unavailable.pysdk.qiniu.com', + 'qiniu.com' + ], + retryCondition: () => false + }), + new OrderRecordMiddleware(recordList, 'A') + ] + }) + .then(({ + _data, + _resp + }) => { + done('this should not be ok'); + }) + .catch(_err => { + recordList.should.eql(['bef_A0']); + done(); + }); + }); + }); +}); diff --git a/test/util.test.js b/test/util.test.js index ef7b4fa..d1f2669 100644 --- a/test/util.test.js +++ b/test/util.test.js @@ -74,6 +74,30 @@ describe('test util functions', function () { }); }); + it('test prepareZone with backup domains', function (done) { + before(function () { + qiniu.conf.UC_HOST = 'fake-uc.nodejssdk.qiniu.com'; + qiniu.conf.UC_BACKUP_HOSTS = [ + 'unavailable-uc.nodejssdk.qiniu.com', + 'uc.qbox.me' + ]; + }); + + after(function () { + qiniu.conf.UC_HOST = 'uc.qbox.me'; + qiniu.conf.UC_BACKUP_HOSTS = [ + 'kodo-config.qiniuapi.com', + 'api.qiniu.com' + ]; + }); + + qiniu.util.prepareZone(bucketManager, bucketManager.mac.accessKey, bucket, function (err, ctx) { + should.not.exist(err); + should.equal(bucketManager, ctx); + done(); + }); + }); + it('test formatDateUTC', function () { const caseList = [ {