Skip to content

Commit

Permalink
More robust configuration options for FetchRequest getUrl functions (#…
Browse files Browse the repository at this point in the history 
…4353).
  • Loading branch information
@ricmoo
ricmoo committed Sep 13, 2023
1 parent c7f8abe commit 9541f2f
Show file tree
Hide file tree
Showing 3 changed files with 182 additions and 105 deletions.
51 changes: 47 additions & 4 deletions src.ts/utils/fetch.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import { assert, assertArgument } from "./errors.js";
import { defineProperties } from "./properties.js";
import { toUtf8Bytes, toUtf8String } from "./utf8.js"

import { getUrl } from "./geturl.js";
import { createGetUrl } from "./geturl.js";

/**
* An environments implementation of ``getUrl`` must return this type.
Expand Down Expand Up @@ -77,7 +77,7 @@ const MAX_ATTEMPTS = 12;
const SLOT_INTERVAL = 250;

// The global FetchGetUrlFunc implementation.
let getUrlFunc: FetchGetUrlFunc = getUrl;
let defaultGetUrlFunc: FetchGetUrlFunc = createGetUrl();

const reData = new RegExp("^data:([^;:]*)?(;base64)?,(.*)$", "i");
const reIpfs = new RegExp("^ipfs:/\/(ipfs/)?(.*)$", "i");
Expand Down Expand Up @@ -201,6 +201,8 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {

#throttle: Required<FetchThrottleParams>;

#getUrlFunc: null | FetchGetUrlFunc;

/**
* The fetch URI to requrest.
*/
Expand Down Expand Up @@ -429,6 +431,28 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
this.#retry = retry;
}

/**
* This function is called to fetch content from HTTP and
* HTTPS URLs and is platform specific (e.g. nodejs vs
* browsers).
*
* This is by default the currently registered global getUrl
* function, which can be changed using [[registerGetUrl]].
* If this has been set, setting is to ``null`` will cause
* this FetchRequest (and any future clones) to revert back to
* using the currently registered global getUrl function.
*
* Setting this is generally not necessary, but may be useful
* for developers that wish to intercept requests or to
* configurege a proxy or other agent.
*/
get getUrlFunc(): FetchGetUrlFunc {
return this.#getUrlFunc || defaultGetUrlFunc;
}
set getUrlFunc(value: null | FetchGetUrlFunc) {
this.#getUrlFunc = value;
}

/**
* Create a new FetchRequest instance with default values.
*
Expand All @@ -448,6 +472,8 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
slotInterval: SLOT_INTERVAL,
maxAttempts: MAX_ATTEMPTS
};

this.#getUrlFunc = null;
}

toString(): string {
Expand Down Expand Up @@ -510,7 +536,7 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
// We have a preflight function; update the request
if (this.preflightFunc) { req = await this.preflightFunc(req); }

const resp = await getUrlFunc(req, checkSignal(_request.#signal));
const resp = await this.getUrlFunc(req, checkSignal(_request.#signal));
let response = new FetchResponse(resp.statusCode, resp.statusMessage, resp.headers, resp.body, _request);

if (response.statusCode === 301 || response.statusCode === 302) {
Expand Down Expand Up @@ -641,6 +667,8 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
clone.#process = this.#process;
clone.#retry = this.#retry;

clone.#getUrlFunc = this.#getUrlFunc;

return clone;
}

Expand Down Expand Up @@ -686,7 +714,22 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
*/
static registerGetUrl(getUrl: FetchGetUrlFunc): void {
if (locked) { throw new Error("gateways locked"); }
getUrlFunc = getUrl;
defaultGetUrlFunc = getUrl;
}

/**
* Creates a getUrl function that fetches content from HTTP and
* HTTPS URLs.
*
* The available %%options%% are dependent on the platform
* implementation of the default getUrl function.
*
* This is not generally something that is needed, but is useful
* when trying to customize simple behaviour when fetching HTTP
* content.
*/
static createGetUrlFunc(options?: Record<string, any>): FetchGetUrlFunc {
return createGetUrl(options);
}

/**
Expand Down
90 changes: 52 additions & 38 deletions src.ts/utils/geturl-browser.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import { assert } from "./errors.js";

import type { FetchRequest, FetchCancelSignal, GetUrlResponse } from "./fetch.js";
import type {
FetchGetUrlFunc, FetchRequest, FetchCancelSignal, GetUrlResponse
} from "./fetch.js";


declare global {
Expand All @@ -27,46 +29,58 @@ declare global {

// @TODO: timeout is completely ignored; start a Promise.any with a reject?

export async function getUrl(req: FetchRequest, _signal?: FetchCancelSignal): Promise<GetUrlResponse> {
const protocol = req.url.split(":")[0].toLowerCase();

assert(protocol === "http" || protocol === "https", `unsupported protocol ${ protocol }`, "UNSUPPORTED_OPERATION", {
info: { protocol },
operation: "request"
});

assert(protocol === "https" || !req.credentials || req.allowInsecureAuthentication, "insecure authorized connections unsupported", "UNSUPPORTED_OPERATION", {
operation: "request"
});

let signal: undefined | AbortSignal = undefined;
if (_signal) {
const controller = new AbortController();
signal = controller.signal;
_signal.addListener(() => { controller.abort(); });
export function createGetUrl(options?: Record<string, any>): FetchGetUrlFunc {

async function getUrl(req: FetchRequest, _signal?: FetchCancelSignal): Promise<GetUrlResponse> {
const protocol = req.url.split(":")[0].toLowerCase();

assert(protocol === "http" || protocol === "https", `unsupported protocol ${ protocol }`, "UNSUPPORTED_OPERATION", {
info: { protocol },
operation: "request"
});

assert(protocol === "https" || !req.credentials || req.allowInsecureAuthentication, "insecure authorized connections unsupported", "UNSUPPORTED_OPERATION", {
operation: "request"
});

let signal: undefined | AbortSignal = undefined;
if (_signal) {
const controller = new AbortController();
signal = controller.signal;
_signal.addListener(() => { controller.abort(); });
}

const init = {
method: req.method,
headers: new Headers(Array.from(req)),
body: req.body || undefined,
signal
};

const resp = await fetch(req.url, init);

const headers: Record<string, string> = { };
resp.headers.forEach((value, key) => {
headers[key.toLowerCase()] = value;
});

const respBody = await resp.arrayBuffer();
const body = (respBody == null) ? null: new Uint8Array(respBody);

return {
statusCode: resp.status,
statusMessage: resp.statusText,
headers, body
};
}

const init = {
method: req.method,
headers: new Headers(Array.from(req)),
body: req.body || undefined,
signal
};

const resp = await fetch(req.url, init);

const headers: Record<string, string> = { };
resp.headers.forEach((value, key) => {
headers[key.toLowerCase()] = value;
});
return getUrl;
}

const respBody = await resp.arrayBuffer();
const body = (respBody == null) ? null: new Uint8Array(respBody);
// @TODO: remove in v7; provided for backwards compat
const defaultGetUrl: FetchGetUrlFunc = createGetUrl({ });

return {
statusCode: resp.status,
statusMessage: resp.statusText,
headers, body
};
export async function getUrl(req: FetchRequest, _signal?: FetchCancelSignal): Promise<GetUrlResponse> {
return defaultGetUrl(req, _signal);
}

146 changes: 83 additions & 63 deletions src.ts/utils/geturl.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,92 +5,112 @@ import { gunzipSync } from "zlib";
import { assert } from "./errors.js";
import { getBytes } from "./data.js";

import type { FetchRequest, FetchCancelSignal, GetUrlResponse } from "./fetch.js";
import type {
FetchGetUrlFunc, FetchRequest, FetchCancelSignal, GetUrlResponse
} from "./fetch.js";

/**
* @_ignore:
*/
export async function getUrl(req: FetchRequest, signal?: FetchCancelSignal): Promise<GetUrlResponse> {
export function createGetUrl(options?: Record<string, any>): FetchGetUrlFunc {

const protocol = req.url.split(":")[0].toLowerCase();
async function getUrl(req: FetchRequest, signal?: FetchCancelSignal): Promise<GetUrlResponse> {

assert(protocol === "http" || protocol === "https", `unsupported protocol ${ protocol }`, "UNSUPPORTED_OPERATION", {
info: { protocol },
operation: "request"
});
const protocol = req.url.split(":")[0].toLowerCase();

assert(protocol === "https" || !req.credentials || req.allowInsecureAuthentication, "insecure authorized connections unsupported", "UNSUPPORTED_OPERATION", {
operation: "request"
});
assert(protocol === "http" || protocol === "https", `unsupported protocol ${ protocol }`, "UNSUPPORTED_OPERATION", {
info: { protocol },
operation: "request"
});

const method = req.method;
const headers = Object.assign({ }, req.headers);
assert(protocol === "https" || !req.credentials || req.allowInsecureAuthentication, "insecure authorized connections unsupported", "UNSUPPORTED_OPERATION", {
operation: "request"
});

const options: any = { method, headers };
const method = req.method;
const headers = Object.assign({ }, req.headers);

const request = ((protocol === "http") ? http: https).request(req.url, options);
const reqOptions: any = { method, headers };
if (options) {
if (options.agent) { reqOptions.agent = options.agent; }
}

request.setTimeout(req.timeout);
const request = ((protocol === "http") ? http: https).request(req.url, reqOptions);

const body = req.body;
if (body) { request.write(Buffer.from(body)); }
request.setTimeout(req.timeout);

request.end();
const body = req.body;
if (body) { request.write(Buffer.from(body)); }

return new Promise((resolve, reject) => {
// @TODO: Node 15 added AbortSignal; once we drop support for
// Node14, we can add that in here too
request.end();

request.once("response", (resp: http.IncomingMessage) => {
const statusCode = resp.statusCode || 0;
const statusMessage = resp.statusMessage || "";
const headers = Object.keys(resp.headers || {}).reduce((accum, name) => {
let value = resp.headers[name] || "";
if (Array.isArray(value)) {
value = value.join(", ");
}
accum[name] = value;
return accum;
}, <{ [ name: string ]: string }>{ });
return new Promise((resolve, reject) => {
// @TODO: Node 15 added AbortSignal; once we drop support for
// Node14, we can add that in here too

let body: null | Uint8Array = null;
//resp.setEncoding("utf8");
request.once("response", (resp: http.IncomingMessage) => {
const statusCode = resp.statusCode || 0;
const statusMessage = resp.statusMessage || "";
const headers = Object.keys(resp.headers || {}).reduce((accum, name) => {
let value = resp.headers[name] || "";
if (Array.isArray(value)) {
value = value.join(", ");
}
accum[name] = value;
return accum;
}, <{ [ name: string ]: string }>{ });

let body: null | Uint8Array = null;
//resp.setEncoding("utf8");

resp.on("data", (chunk: Uint8Array) => {
if (signal) {
try {
signal.checkSignal();
} catch (error) {
return reject(error);
}
}

resp.on("data", (chunk: Uint8Array) => {
if (signal) {
try {
signal.checkSignal();
} catch (error) {
return reject(error);
if (body == null) {
body = chunk;
} else {
const newBody = new Uint8Array(body.length + chunk.length);
newBody.set(body, 0);
newBody.set(chunk, body.length);
body = newBody;
}
}

if (body == null) {
body = chunk;
} else {
const newBody = new Uint8Array(body.length + chunk.length);
newBody.set(body, 0);
newBody.set(chunk, body.length);
body = newBody;
}
});
});

resp.on("end", () => {
if (headers["content-encoding"] === "gzip" && body) {
body = getBytes(gunzipSync(body));
}
resp.on("end", () => {
if (headers["content-encoding"] === "gzip" && body) {
body = getBytes(gunzipSync(body));
}

resolve({ statusCode, statusMessage, headers, body });
});
resolve({ statusCode, statusMessage, headers, body });
});

resp.on("error", (error) => {
//@TODO: Should this just return nornal response with a server error?
(<any>error).response = { statusCode, statusMessage, headers, body };
reject(error);
resp.on("error", (error) => {
//@TODO: Should this just return nornal response with a server error?
(<any>error).response = { statusCode, statusMessage, headers, body };
reject(error);
});
});

request.on("error", (error) => { reject(error); });
});
}

return getUrl;
}

// @TODO: remove in v7; provided for backwards compat
const defaultGetUrl: FetchGetUrlFunc = createGetUrl({ });

request.on("error", (error) => { reject(error); });
});
/**
* @_ignore:
*/
export async function getUrl(req: FetchRequest, signal?: FetchCancelSignal): Promise<GetUrlResponse> {
return defaultGetUrl(req, signal);
}

0 comments on commit 9541f2f

Please sign in to comment.