Skip to content

Commit

Permalink
fix: node-fetch v3 (Fixes #1628 Fixes #970 )
Browse files Browse the repository at this point in the history
  • Loading branch information
@danielweck
danielweck committed Feb 11, 2022
1 parent 2baa605 commit 8f356c2
Show file tree
Hide file tree
Showing 11 changed files with 344 additions and 292 deletions.
497 changes: 286 additions & 211 deletions package-lock.json
View file

Large diffs are not rendered by default.

6 changes: 2 additions & 4 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@
"mathjax": "^3.2.0",
"mime-types": "^2.1.34",
"nanoid": "^3.2.0",
"node-fetch": "^2.6.7",
"node-fetch": "^3.2.0",
"opds-feed-parser": "0.0.18",
"r2-lcp-js": "^1.0.35",
"r2-navigator-js": "^1.9.13",
Expand All @@ -279,9 +279,9 @@
"redux-first-history": "^5.0.8",
"redux-saga": "^1.1.3",
"reflect-metadata": "^0.1.13",
"request": "^2.88.2",
"rfc6902": "^5.0.0",
"ta-json-x": "^2.5.3",
"timeout-signal": "^1.1.0",
"tmp": "^0.2.1",
"typed-redux-saga": "^1.3.1",
"uuid": "^8.3.2",
Expand All @@ -305,7 +305,6 @@
"@types/lunr": "^2.3.4",
"@types/mime-types": "^2.1.1",
"@types/node": "^16.11.22",
"@types/node-fetch": "^2.5.12",
"@types/ramda": "^0.27.64",
"@types/react": "^17.0.39",
"@types/react-beautiful-dnd": "^13.1.2",
Expand All @@ -315,7 +314,6 @@
"@types/react-router-dom": "^5.3.3",
"@types/redux": "^3.6.31",
"@types/remote-redux-devtools": "^0.5.5",
"@types/request": "^2.48.8",
"@types/tmp": "^0.2.3",
"@types/urijs": "^1.19.18",
"@types/uuid": "^8.3.4",
Expand Down
2 changes: 1 addition & 1 deletion src/common/utils/http.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

import { RequestInit, Response } from "node-fetch";

export type THttpOptions = RequestInit;
export type THttpOptions = RequestInit & { timeout?: number, abortController?: AbortController };
export type THttpResponse = Response;

export interface IHttpGetResult<TData> {
Expand Down
98 changes: 38 additions & 60 deletions src/main/network/http.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@
// that can be found in the LICENSE file exposed on Github (readium) in the project repository.
// ==LICENSE-END==

import timeoutSignal from "timeout-signal";
import * as debug_ from "debug";
import { promises as fsp } from "fs";
import * as http from "http";
import * as https from "https";
import { Headers, RequestInit } from "node-fetch";
import { AbortSignal as IAbortSignal } from "node-fetch/externals";
import { AbortError, Headers, RequestInit, Response } from "node-fetch";
import {
IHttpGetResult, THttpGetCallback, THttpOptions, THttpResponse,
} from "readium-desktop/common/utils/http";
Expand Down Expand Up @@ -168,8 +168,8 @@ export async function httpFetchRawResponse(
options.headers = options.headers instanceof Headers
? options.headers
: new Headers(options.headers || {});
options.headers.set("user-agent", "readium-desktop");
options.headers.set("accept-language", `${locale},en-US;q=0.7,en;q=0.5`);
(options.headers as Headers).set("user-agent", "readium-desktop");
(options.headers as Headers).set("accept-language", `${locale},en-US;q=0.7,en;q=0.5`);
options.redirect = "manual"; // handle cookies

// https://github.com/node-fetch/node-fetch#custom-agent
Expand Down Expand Up @@ -199,7 +199,26 @@ export async function httpFetchRawResponse(
// }
options.timeout = options.timeout || DEFAULT_HTTP_TIMEOUT;

const response = await fetchWithCookie(url, options);
let timeout: NodeJS.Timeout | undefined;
if (!options.signal) {
options.signal = timeoutSignal(options.timeout);
} else if (options.abortController) {
timeout = setTimeout(() => {
timeout = undefined;
try {
options.abortController.abort();
} catch {}
}, options.timeout);
}

let response: Response;
try {
response = await fetchWithCookie(url.toString(), options);
} finally {
if (timeout) {
clearTimeout(timeout);
}
}

debug("fetch URL:", `${url}`);
debug("Method", options.method);
Expand Down Expand Up @@ -234,7 +253,7 @@ export async function httpFetchRawResponse(
if (!(options.headers instanceof Headers)) {
options.headers = new Headers(options.headers);
}
options.headers.delete("content-length");
(options.headers as Headers).delete("content-length");
}
}

Expand Down Expand Up @@ -304,24 +323,24 @@ export async function httpFetchFormattedResponse<TData = undefined>(
debug("url: ", url);
debug("options: ", options);

if (err.name === "AbortError") {
if (errStr.includes("timeout")) { // err.name === "FetchError"
result = {
isAbort: true,
isNetworkError: false,
isTimeout: false,
isAbort: false,
isNetworkError: true,
isTimeout: true,
isFailure: true,
isSuccess: false,
url,
statusMessage: errStr,
};
} else if (errStr.includes("timeout")) { // err.name === "FetchError"
} else if (err.name === "AbortError" || err instanceof AbortError) {
result = {
isAbort: false,
isNetworkError: true,
isTimeout: true,
isAbort: true,
isNetworkError: false,
isTimeout: false,
isFailure: true,
isSuccess: false,
url,
statusMessage: errStr,
};
} else { // err.name === "FetchError"
result = {
Expand Down Expand Up @@ -414,7 +433,7 @@ const httpGetUnauthorized =
? options.headers
: new Headers(options.headers || {});

options.headers.set("Authorization", httpSetHeaderAuthorization(tokenType || "Bearer", accessToken));
(options.headers as Headers).set("Authorization", httpSetHeaderAuthorization(tokenType || "Bearer", accessToken));

const response = await httpGetWithAuth(false)(
url,
Expand All @@ -435,7 +454,7 @@ const httpGetUnauthorized =
// In some cases the returned content won't launch a new authentication process
// It's safer to just delete the access token and start afresh now.
await deleteAuthenticationToken(url.host);
options.headers.delete("Authorization");
(options.headers as Headers).delete("Authorization");
const responseWithoutAuth = await httpGetWithAuth(
false,
)(url, options, _callback, ..._arg);
Expand All @@ -457,7 +476,7 @@ const httpGetUnauthorizedRefresh =
options.headers = options.headers instanceof Headers
? options.headers
: new Headers(options.headers || {});
options.headers.set("Content-Type", "application/json");
(options.headers as Headers).set("Content-Type", "application/json");

options.body = JSON.stringify({
refresh_token: refreshToken,
Expand All @@ -466,7 +485,7 @@ const httpGetUnauthorizedRefresh =

const httpPostResponse = await httpPost(refreshUrl, options);
if (httpPostResponse.isSuccess) {
const jsonDataResponse = await httpPostResponse.response.json();
const jsonDataResponse: any = await httpPostResponse.response.json();

const newRefreshToken = typeof jsonDataResponse?.refresh_token === "string"
? jsonDataResponse.refresh_token
Expand Down Expand Up @@ -507,44 +526,3 @@ export const httpPost: typeof httpFetchFormattedResponse =

return httpFetchFormattedResponse(...arg);
};

// fetch checks the class name
// https://github.com/node-fetch/node-fetch/blob/b7076bb24f75be688d8fc8b175f41b341e853f2b/src/utils/is.js#L78
export class AbortSignal implements IAbortSignal {

public aborted: boolean;
private listenerArray: any[];

constructor() {
this.listenerArray = [];
this.aborted = false;
}

public onabort: IAbortSignal["onabort"] = null;

// public get aborted() {
// return this._aborted;
// }

public addEventListener(_type: "abort", listener: (a: any[]) => any) {
this.listenerArray.push(listener);
}

public removeEventListener(_type: "abort", listener: (a: any[]) => any) {
const index = this.listenerArray.findIndex((v) => v === listener);
if (index > -1) {
this.listenerArray = [...this.listenerArray.slice(0, index), ...this.listenerArray.slice(index + 1)];
}
}

public dispatchEvent() {
this.listenerArray.forEach((l) => {
try {
l();
} catch (_e) {
// ignore
}
});
return this.aborted = true;
}
}
4 changes: 2 additions & 2 deletions src/main/redux/sagas/api/publication/import/importFromLink.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
// ==LICENSE-END==

import * as debug_ from "debug";
import fetch from "node-fetch";
import nodeFetch from "node-fetch";
import { IOpdsLinkView, IOpdsPublicationView } from "readium-desktop/common/views/opds";
import { PublicationDocument } from "readium-desktop/main/db/document/publication";
import { diMainGet } from "readium-desktop/main/di";
Expand Down Expand Up @@ -86,7 +86,7 @@ export function* importFromLinkService(

if (!link.type) {
try {
const response = yield* callTyped(() => fetch(url));
const response = yield* callTyped(() => nodeFetch(url.toString()));
const contentType = response?.headers?.get("Content-Type");
if (contentType) {
link.type = contentType;
Expand Down
2 changes: 1 addition & 1 deletion src/main/redux/sagas/auth.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,7 @@ async function opdsSetAuthCredentials(
async (res) => {
if (res.isSuccess) {

const _data = await res.response.json();
const _data: any = await res.response.json();
if (typeof _data === "object") {

res.data = {
Expand Down
19 changes: 10 additions & 9 deletions src/main/redux/sagas/downloader.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,15 @@

import * as debug_ from "debug";
import { createWriteStream, promises as fsp, WriteStream } from "fs";
import { RequestInit } from "node-fetch";
import * as path from "path";
import { acceptedExtension } from "readium-desktop/common/extension";
import { ToastType } from "readium-desktop/common/models/toast";
import { downloadActions, toastActions } from "readium-desktop/common/redux/actions";
import { ok } from "readium-desktop/common/utils/assert";
import { IHttpGetResult } from "readium-desktop/common/utils/http";
import { IHttpGetResult, THttpOptions } from "readium-desktop/common/utils/http";
import { diMainGet } from "readium-desktop/main/di";
import { createTempDir } from "readium-desktop/main/fs/path";
import { AbortSignal, httpGet } from "readium-desktop/main/network/http";
import { httpGet } from "readium-desktop/main/network/http";
import { findExtWithMimeType } from "readium-desktop/utils/mimeTypes";
// eslint-disable-next-line local-rules/typed-redux-saga-use-typed-effects
import { cancel, cancelled, delay, take } from "redux-saga/effects";
Expand Down Expand Up @@ -208,10 +207,11 @@ function* downloaderServiceProcessStatusProgressLoop(
}
}

function* downloadLinkRequest(linkHref: string, abort: AbortSignal): SagaGenerator<IHttpGetResult<undefined>> {
function* downloadLinkRequest(linkHref: string, controller: AbortController): SagaGenerator<IHttpGetResult<undefined>> {

const options: RequestInit = {};
options.signal = abort;
const options: THttpOptions = {};
options.abortController = controller;
options.signal = controller.signal;

const data = yield* callTyped(() => httpGet(linkHref, options));

Expand Down Expand Up @@ -396,14 +396,15 @@ type TReturnDownloadLinkProcess = TReturnDownloadLinkStream | undefined;
function* downloadLinkProcess(linkHref: IDownloaderLink, id: number): SagaGenerator<TReturnDownloadLinkProcess> {

ok(linkHref);
const abort = new AbortSignal();

const controller = new AbortController();

try {

debug("start to downloadService", linkHref);
const url = typeof linkHref === "string" ? linkHref : linkHref.href;
const type = typeof linkHref === "string" ? undefined : linkHref.type;
const httpData = yield* callTyped(downloadLinkRequest, url, abort);
const httpData = yield* callTyped(downloadLinkRequest, url, controller);

debug("start to stream download");
return yield* callTyped(downloadLinkStream, httpData, id, type);
Expand All @@ -415,7 +416,7 @@ function* downloadLinkProcess(linkHref: IDownloaderLink, id: number): SagaGenera
if (yield cancelled()) {
debug("downloaderService cancelled -> abort");

abort.dispatchEvent();
controller.abort();
}

}
Expand Down
2 changes: 1 addition & 1 deletion webpack.config.main.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ const _externalsCache = new Set();
if (nodeEnv !== "production") {
const nodeExternals = require("webpack-node-externals");
const neFunc = nodeExternals({
allowlist: ["normalize-url"],
allowlist: ["normalize-url", "node-fetch", "data-uri-to-buffer", /^fetch-blob/, /^formdata-polyfill/],
importType: function (moduleName) {
if (!_externalsCache.has(moduleName)) {
console.log(`WEBPACK EXTERNAL (MAIN): [${moduleName}]`);
Expand Down
2 changes: 1 addition & 1 deletion webpack.config.renderer-library.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ const _externalsCache = new Set();
if (nodeEnv !== "production") {
const nodeExternals = require("webpack-node-externals");
const neFunc = nodeExternals({
allowlist: ["normalize-url"],
allowlist: ["normalize-url", "node-fetch", "data-uri-to-buffer", /^fetch-blob/, /^formdata-polyfill/],
importType: function (moduleName) {
if (!_externalsCache.has(moduleName)) {
console.log(`WEBPACK EXTERNAL (LIBRARY): [${moduleName}]`);
Expand Down
2 changes: 1 addition & 1 deletion webpack.config.renderer-pdf.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ const _externalsCache = new Set();
if (nodeEnv !== "production") {
const nodeExternals = require("webpack-node-externals");
const neFunc = nodeExternals({
allowlist: ["normalize-url"],
allowlist: ["normalize-url", "node-fetch", "data-uri-to-buffer", /^fetch-blob/, /^formdata-polyfill/],
importType: function (moduleName) {
if (!_externalsCache.has(moduleName)) {
console.log(`WEBPACK EXTERNAL (PDF): [${moduleName}]`);
Expand Down
2 changes: 1 addition & 1 deletion webpack.config.renderer-reader.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ const _externalsCache = new Set();
if (nodeEnv !== "production") {
const nodeExternals = require("webpack-node-externals");
const neFunc = nodeExternals({
allowlist: ["normalize-url"],
allowlist: ["normalize-url", "node-fetch", "data-uri-to-buffer", /^fetch-blob/, /^formdata-polyfill/],
importType: function (moduleName) {
if (!_externalsCache.has(moduleName)) {
console.log(`WEBPACK EXTERNAL (READER): [${moduleName}]`);
Expand Down

0 comments on commit 8f356c2

Please sign in to comment.