Skip to content

Commit

Permalink
📦 NEW: Support upload file by args.files
Browse files Browse the repository at this point in the history
  • Loading branch information
@fengmk2
fengmk2 committed Jul 5, 2022
1 parent 70a238d commit c3cc765
Show file tree
Hide file tree
Showing 24 changed files with 1,267 additions and 669 deletions.
12 changes: 8 additions & 4 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
],
"author": "fengmk2 <fengmk2@gmail.com> (https://github.com/fengmk2)",
"homepage": "https://github.com/node-modules/urllib",
"main": "dist/index.js",
"main": "./dist/index.js",
"files": [
"dist",
"src"
Expand All @@ -26,8 +26,8 @@
},
"scripts": {
"lint": "eslint src test --ext .ts",
"build": "npm run build:dist && node -p \"require('.')\"",
"build:dist": "tsc --version && tsc -p ./tsconfig.build.json",
"build": "npm run build:dist && node test/cjs/index.js",
"build:dist": "tsc --version && rm -rf ./dist *.tsbuildinfo && tsc -p ./tsconfig.build.json",
"test": "tsc --version && jest --coverage",
"ci": "npm run lint && npm run test && npm run build",
"contributor": "git-contributor"
Expand All @@ -36,16 +36,20 @@
"content-type": "^1.0.2",
"default-user-agent": "^1.0.0",
"digest-header": "^0.0.1",
"formstream": "^1.1.0",
"formdata-node": "^4.3.3",
"humanize-ms": "^1.2.0",
"iconv-lite": "^0.4.15",
"ip": "^1.1.5",
"mime-types": "^2.1.35",
"tslib": "^2.4.0",
"undici": "^5.4.0"
},
"devDependencies": {
"@types/busboy": "^1.5.0",
"@types/default-user-agent": "^1.0.0",
"@types/jest": "28",
"@types/mime-types": "^2.1.1",
"busboy": "^1.6.0",
"coffee": "5",
"egg-ci": "2",
"eslint": "^8.17.0",
Expand Down
175 changes: 154 additions & 21 deletions src/HttpClient.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,14 @@
import { EventEmitter } from 'events';
import { debuglog } from 'util';
import { fetch, Request, RequestInit, Headers } from 'undici';
import { Readable, isReadable } from 'stream';
import { Blob } from 'buffer';
import { basename } from 'path';
import {
fetch, RequestInit, Headers, FormData,
} from 'undici';
import { fileFromPath } from 'formdata-node/file-from-path';
import createUserAgent from 'default-user-agent';
import mime from 'mime-types';
import { RequestURL, RequestOptions } from './Request';

const debug = debuglog('urllib');
Expand All @@ -10,17 +17,47 @@ export type ClientOptions = {
defaultArgs?: RequestOptions;
};

// https://github.com/octet-stream/form-data
class BlobFromStream {
#stream;
#type;
constructor(stream: Readable, type: string) {
this.#stream = stream;
this.#type = type;
}

stream() {
return this.#stream;
}

get type(): string {
return this.#type;
}

get [Symbol.toStringTag]() {
return 'Blob';
}
}

class HttpClientRequestTimeoutError extends Error {
constructor(timeout: number, cause: Error) {
constructor(timeout: number, options: ErrorOptions) {
const message = `Request timeout for ${timeout} ms`;
super(message, { cause });
super(message, options);
this.name = this.constructor.name;
Error.captureStackTrace(this, this.constructor);
}
}

const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.0.0');

function getFileName(stream: Readable) {
const filePath: string = (stream as any).path;
if (filePath) {
return basename(filePath);
}
return '';
}

export class HttpClient extends EventEmitter {
defaultArgs?: RequestOptions;

Expand All @@ -30,6 +67,7 @@ export class HttpClient extends EventEmitter {
}

async request(url: RequestURL, options?: RequestOptions) {
const requestUrl = typeof url === 'string' ? new URL(url) : url;
const args = {
...this.defaultArgs,
...options,
Expand Down Expand Up @@ -65,8 +103,10 @@ export class HttpClient extends EventEmitter {
requestTimeout = args.timeout;
}
}

const requestTimeoutController = new AbortController();
const requestTimerId = setTimeout(() => requestTimeoutController.abort(), requestTimeout);
const method = (args.method ?? 'GET').toUpperCase();

try {
const headers = new Headers(args.headers ?? {});
Expand All @@ -82,42 +122,134 @@ export class HttpClient extends EventEmitter {
}

const requestOptions: RequestInit = {
method: args.method ?? 'GET',
method,
keepalive: true,
headers,
signal: requestTimeoutController.signal,
};
if (args.followRedirect === false) {
requestOptions.redirect = 'manual';
}

debug('%s %j, headers: %j, timeout: %s', requestOptions.method, url, args.headers, requestTimeout);
const request = new Request(url, requestOptions);
const isGETOrHEAD = requestOptions.method === 'GET' || requestOptions.method === 'HEAD';

if (args.files) {
if (isGETOrHEAD) {
requestOptions.method = 'POST';
}
const formData = new FormData();
const uploadFiles: [string, string | Readable | Buffer][] = [];
if (Array.isArray(args.files)) {
for (const [ index, file ] of args.files.entries()) {
const field = index === 0 ? 'file' : `file${index}`;
uploadFiles.push([ field, file ]);
}
} else if (args.files instanceof Readable || isReadable(args.files as any)) {
uploadFiles.push([ 'file', args.files as Readable ]);
} else if (typeof args.files === 'string' || Buffer.isBuffer(args.files)) {
uploadFiles.push([ 'file', args.files ]);
} else if (typeof args.files === 'object') {
for (const field in args.files) {
uploadFiles.push([ field, args.files[field] ]);
}
}
// set normal fields first
if (args.data) {
for (const field in args.data) {
formData.append(field, args.data[field]);
}
}
for (const [ index, [ field, file ]] of uploadFiles.entries()) {
if (typeof file === 'string') {
// FIXME: support non-ascii filename
// const fileName = encodeURIComponent(basename(file));
// formData.append(field, await fileFromPath(file, `utf-8''${fileName}`, { type: mime.lookup(fileName) || '' }));
const fileName = basename(file);
formData.append(field, await fileFromPath(file, fileName, { type: mime.lookup(fileName) || '' }));
} else if (Buffer.isBuffer(file)) {
formData.append(field, new Blob([ file ]), `bufferfile${index}`);
} else if (file instanceof Readable || isReadable(file as any)) {
const fileName = getFileName(file) || `streamfile${index}`;
formData.append(field, new BlobFromStream(file, mime.lookup(fileName) || ''), fileName);
}
}
requestOptions.body = formData;
} else if (args.content) {
if (!isGETOrHEAD) {
if (isReadable(args.content as Readable)) {
// disable keepalive
requestOptions.keepalive = false;
}
// handle content
requestOptions.body = args.content;
if (args.contentType) {
headers.set('content-type', args.contentType);
}
}
} else if (args.data) {
const isStringOrBufferOrReadable = typeof args.data === 'string'
|| Buffer.isBuffer(args.data)
|| isReadable(args.data);
if (isGETOrHEAD) {
if (!isStringOrBufferOrReadable) {
for (const field in args.data) {
requestUrl.searchParams.append(field, args.data[field]);
}
}
} else {
if (isStringOrBufferOrReadable) {
if (isReadable(args.data as Readable)) {
// disable keepalive
requestOptions.keepalive = false;
}
requestOptions.body = args.data;
} else {
if (args.contentType === 'json'
|| args.contentType === 'application/json'
|| headers.get('content-type')?.startsWith('application/json')) {
requestOptions.body = JSON.stringify(args.data);
if (!headers.has('content-type')) {
headers.set('content-type', 'application/json');
}
} else {
requestOptions.body = new URLSearchParams(args.data);
}
}
}
}

debug('%s %s, headers: %j, timeout: %s', requestOptions.method, url, headers, requestTimeout);
requestOptions.headers = headers;

const response = await fetch(requestUrl, requestOptions);
for (const [ name, value ] of response.headers) {
res.headers[name] = value;
}
res.status = res.statusCode = response.status;
res.statusMessage = response.statusText;
if (response.redirected) {
res.requestUrls.push(response.url);
}
if (res.headers['content-length']) {
res.size = parseInt(res.headers['content-length']);
}

const response = await fetch(request);
let data: any;
if (args.streaming || args.dataType === 'stream') {
data = response.body;
} else if (args.dataType === 'text') {
data = await response.text();
} else if (args.dataType === 'json') {
data = await response.json();
if (requestOptions.method === 'HEAD') {
data = {};
} else {
data = await response.json();
}
} else {
// buffer
data = Buffer.from(await response.arrayBuffer());
}
for (const [ name, value ] of response.headers) {
res.headers[name] = value;
}
res.status = res.statusCode = response.status;
res.statusMessage = response.statusText;
res.rt = res.timing.contentDownload = Date.now() - requestStartTime;
if (res.headers['content-length']) {
res.size = parseInt(res.headers['content-length']);
}
if (response.redirected) {
res.requestUrls.push(response.url);
}

return {
status: res.status,
data,
Expand All @@ -129,9 +261,10 @@ export class HttpClient extends EventEmitter {
} catch (e: any) {
let err = e;
if (requestTimeoutController.signal.aborted) {
err = new HttpClientRequestTimeoutError(requestTimeout, e as Error);
err = new HttpClientRequestTimeoutError(requestTimeout, { cause: e });
}
err.res = res;
// console.error(err);
throw err;
} finally {
clearTimeout(requestTimerId);
Expand Down
11 changes: 3 additions & 8 deletions src/Request.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,13 @@ export type RequestURL = string | URL;

export type RequestOptions = {
/** Request method, defaults to GET. Could be GET, POST, DELETE or PUT. Alias 'type'. */
method?: HttpMethod;
method?: HttpMethod | Lowercase<HttpMethod>;
/** Data to be sent. Will be stringify automatically. */
data?: any;
/** Force convert data to query string. */
dataAsQueryString?: boolean;
/** Manually set the content of payload. If set, data will be ignored. */
content?: string | Buffer;
content?: string | Buffer | Readable;
/** Stream to be pipe to the remote. If set, data and content will be ignored. */
stream?: Readable;
/**
Expand All @@ -28,14 +28,9 @@ export type RequestOptions = {
* The files will send with multipart/form-data format, base on formstream.
* If method not set, will use POST method by default.
*/
files?: Array<Readable | Buffer | string> | object | Readable | Buffer | string;
files?: Array<Readable | Buffer | string> | Record<string, Readable | Buffer | string> | Readable | Buffer | string;
/** Type of request data, could be 'json'. If it's 'json', will auto set Content-Type: 'application/json' header. */
contentType?: string;
/**
* urllib default use querystring to stringify form data which don't support nested object,
* will use qs instead of querystring to support nested object by set this option to true.
*/
nestedQuerystring?: boolean;
/**
* Type of response data. Could be text or json.
* If it's text, the callbacked data would be a String.
Expand Down

0 comments on commit c3cc765

Please sign in to comment.