Skip to content

Commit

Permalink
switch from experimental native fetch to axios
Browse files Browse the repository at this point in the history
  • Loading branch information
@majodev
majodev committed May 29, 2023
1 parent b42b8eb commit 1eaf40a
Show file tree
Hide file tree
Showing 8 changed files with 128 additions and 70 deletions.
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
"url": "https://github.com/majodev/google-webfonts-helper.git"
},
"dependencies": {
"axios": "^1.4.0",
"bluebird": "3.7.2",
"compression": "1.7.4",
"css": "3.0.0",
Expand Down
51 changes: 26 additions & 25 deletions server/api/fonts.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ describe("GET /api/fonts/:id", () => {

should(getStats().urlMap).eql(1);
should(getStats().archiveMap).eql(0);
});
}).timeout(10000);

it("should respond with font files for istok-web multi charsets filtered", async () => {
const res = await request(app)
Expand Down Expand Up @@ -147,15 +147,16 @@ describe("GET /api/fonts/:id", () => {

should(getStats().urlMap).eql(1);
should(getStats().archiveMap).eql(0);
});
}).timeout(10000);

it("should respond with 200 for known font istok-web empty subsets", async () => {
it("should respond with 200 for known font istok-web empty subsets", async function () {
this.timeout(10000);
const res = await request(app).get("/api/fonts/istok-web?subsets=").timeout(10000).expect(200).expect("Content-Type", /json/);
should(res.body).be.instanceof(Object);

should(getStats().urlMap).eql(1);
should(getStats().archiveMap).eql(0);
});
}).timeout(10000);

it("should respond with 404 for unknown font", async () => {
await request(app)
Expand All @@ -166,7 +167,7 @@ describe("GET /api/fonts/:id", () => {

should(getStats().urlMap).eql(0);
should(getStats().archiveMap).eql(0);
});
}).timeout(10000);

it("should respond with 404 for unknown font and subset", async () => {
await request(app)
Expand All @@ -177,15 +178,15 @@ describe("GET /api/fonts/:id", () => {

should(getStats().urlMap).eql(0);
should(getStats().archiveMap).eql(0);
});
}).timeout(10000);

it("should respond with 404 for known font istok-web and unknown subset", async () => {
await request(app)
.get("/api/fonts/istok-web?subsets=unknownsubset")
.timeout(10000)
.expect(404)
.expect("Content-Type", /text\/html/);
});
}).timeout(10000);
});

describe("GET /api/fonts/:id?download=zip", () => {
Expand Down Expand Up @@ -219,7 +220,7 @@ describe("GET /api/fonts/:id?download=zip", () => {

should(getStats().urlMap).eql(1);
should(getStats().archiveMap).eql(1);
});
}).timeout(10000);

it("should (concurrently) download istok-web (subsets and formats mix)", async function () {
this.timeout(10000);
Expand Down Expand Up @@ -262,7 +263,7 @@ describe("GET /api/fonts/:id?download=zip", () => {

// 60 files in archive2
should(_.keys(archive2.files).length).eql(20);
});
}).timeout(10000);

it("should (concurrently) download playfair-display (different but unknown subsets resolve to the same key)", async function () {
let triggered = 0;
Expand Down Expand Up @@ -300,14 +301,14 @@ describe("GET /api/fonts/:id?download=zip", () => {

const archive1 = await JSZip.loadAsync(<Buffer>res1.body);

// 59 files in archive1
should(_.keys(archive1.files).length).eql(59);
// 60 files in archive1
should(_.keys(archive1.files).length).eql(60);

const archive2 = await JSZip.loadAsync(<Buffer>res2.body);

// 59 files in archive2
should(_.keys(archive2.files).length).eql(59);
});
// 60 files in archive2
should(_.keys(archive2.files).length).eql(60);
}).timeout(10000);

it("should respond with 200 for download attempt of known font istok-web with unspecified subset", async function () {
this.timeout(10000);
Expand Down Expand Up @@ -347,7 +348,7 @@ describe("GET /api/fonts/:id?download=zip", () => {
should((await fileTypeFromBuffer(await files[6].async("nodebuffer")))?.mime).eql("font/woff");
should(files[7].name).eql("istok-web-v20-latin-regular.woff2");
should((await fileTypeFromBuffer(await files[7].async("nodebuffer")))?.mime).eql("font/woff2");
});
}).timeout(10000);

it("should respond with 200 for download attempt of known font istok-web with unspecified formats", async () => {
const res = await request(app)
Expand Down Expand Up @@ -412,7 +413,7 @@ describe("GET /api/fonts/:id?download=zip", () => {
should((await fileTypeFromBuffer(await files[18].async("nodebuffer")))?.mime).eql("font/woff");
should(files[19].name).eql("istok-web-v20-latin-regular.woff2");
should((await fileTypeFromBuffer(await files[19].async("nodebuffer")))?.mime).eql("font/woff2");
});
}).timeout(10000);

it("should respond with 200 for download attempt of known font istok-web and empty subsets", async () => {
const res = await request(app)
Expand All @@ -433,7 +434,7 @@ describe("GET /api/fonts/:id?download=zip", () => {
_.each(_.sortBy(_.keys(archive.files)), (key) => {
should(key.indexOf("istok-web-v20-latin-")).eql(0);
});
});
}).timeout(10000);

it("should respond with 200 for download attempt of known font istok-web and a single unknown format sneaked in", async () => {
const res = await request(app)
Expand All @@ -454,7 +455,7 @@ describe("GET /api/fonts/:id?download=zip", () => {
_.each(_.sortBy(_.keys(archive.files)), (key) => {
should(key.indexOf("istok-web-v20-latin-")).eql(0);
});
});
}).timeout(10000);

it("should respond with 200 for download attempt of known font istok-web with variants", async () => {
const res = await request(app)
Expand All @@ -476,7 +477,7 @@ describe("GET /api/fonts/:id?download=zip", () => {
should(_.endsWith(key, ".woff") || _.endsWith(key, ".woff2")).eql(true);
should(key.indexOf("regular") === -1).eql(false);
});
});
}).timeout(10000);

it("should respond with 200 for download attempt of known font istok-web with one known, one unknown variant", async () => {
const res = await request(app)
Expand All @@ -498,15 +499,15 @@ describe("GET /api/fonts/:id?download=zip", () => {
should(_.endsWith(key, ".woff") || _.endsWith(key, ".woff2")).eql(true);
should(key.indexOf("regular") === -1).eql(false);
});
});
}).timeout(10000);

it("should respond with 404 for download attempt of known font istok-web with empty variants", async () => {
await request(app)
.get("/api/fonts/istok-web?download=zip&formats=woff,woff2&variants=")
.timeout(10000)
.expect(404)
.expect("Content-Type", /text\/html/);
});
}).timeout(10000);

// https://gwfh.mranftl.com/api/fonts/siemreap?download=zip&subsets=latin,latin-ext&formats=eot,woff,woff2,svg,ttf
it("should respond with 404 for download attempt of unknown font and unknown subset", async () => {
Expand All @@ -515,29 +516,29 @@ describe("GET /api/fonts/:id?download=zip", () => {
.timeout(10000)
.expect(404)
.expect("Content-Type", /text\/html/);
});
}).timeout(10000);

it("should respond with 404 for download attempt of known font istok-web and unknown subset", async () => {
await request(app)
.get("/api/fonts/istok-web?download=zip&subsets=unknown&formats=woff,woff2")
.timeout(10000)
.expect(404)
.expect("Content-Type", /text\/html/);
});
}).timeout(10000);

it("should respond with 404 for download attempt of known font istok-web and unknown format", async () => {
await request(app)
.get("/api/fonts/istok-web?download=zip&formats=rolf")
.timeout(10000)
.expect(404)
.expect("Content-Type", /text\/html/);
});
}).timeout(10000);

it("should respond with 404 for download attempt of known font istok-web and empty formats", async () => {
await request(app)
.get("/api/fonts/istok-web?download=zip&formats=")
.timeout(10000)
.expect(404)
.expect("Content-Type", /text\/html/);
});
}).timeout(10000);
});
2 changes: 1 addition & 1 deletion server/api/healthy.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ describe("GET /-/healthy", () => {
it("should respond with 200", async () => {
await request(app)
.get("/-/healthy")
.timeout(2000)
.timeout(4000)
.expect(200)
.expect("Content-Type", /text\/plain/);
});
Expand Down
4 changes: 2 additions & 2 deletions server/app.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ describe("GET /api/not_defined", () => {
it("should respond with 404", async () => {
await request(app)
.get("/api/not_defined")
.timeout(2000)
.timeout(4000)
.expect(404)
.expect("Content-Type", /text\/html/);
});
Expand All @@ -15,7 +15,7 @@ describe("GET /", () => {
it("should respond with 200", async () => {
await request(app)
.get("/")
.timeout(2000)
.timeout(4000)
.expect(200)
.expect("Content-Type", /text\/html/);
});
Expand Down
37 changes: 26 additions & 11 deletions server/logic/fetchCSS.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@ import * as css from "css";
import * as _ from "lodash";
import { IUserAgents } from "../config";
import { asyncRetry } from "../utils/asyncRetry";
import axios from "axios";

const RETRIES = 2;
const REQUEST_TIMEOUT_MS = 6000;

interface IResource {
src: string | null;
Expand All @@ -20,24 +22,37 @@ export async function fetchCSS(family: string, cssSubsetString: string, type: ke

const txt = await asyncRetry(
async () => {
const res = await fetch(url, {

const res = await axios.get<string>(url, {
timeout: REQUEST_TIMEOUT_MS,
responseType: "text",
maxRedirects: 0, // https://github.com/axios/axios/issues/2610
headers: {
accept: "text/css,*/*;q=0.1",
Accept: "text/css,*/*;q=0.1",
"User-Agent": userAgent,
},
}
});

if (res.status !== 200) {
throw new Error(`${url} fetchCSS request failed. status code: ${res.status} ${res.statusText}`);
}
return res.data;

const contentType = res.headers.get("content-type");
// const res = await fetch(url, {
// headers: {
// accept: "text/css,*/*;q=0.1",
// "User-Agent": userAgent,
// },
// });

if (_.isNil(contentType) || _.isEmpty(contentType) || contentType.indexOf("css") === -1) {
throw new Error(`${url} fetchCSS request failed. expected "css" to be in content-type header: ${contentType}`);
}
// if (res.status !== 200) {
// throw new Error(`${url} fetchCSS request failed. status code: ${res.status} ${res.statusText}`);
// }

// const contentType = res.headers.get("content-type");

// if (_.isNil(contentType) || _.isEmpty(contentType) || contentType.indexOf("css") === -1) {
// throw new Error(`${url} fetchCSS request failed. expected "css" to be in content-type header: ${contentType}`);
// }

return res.text();
// return res.text();
},
{ retries: RETRIES }
);
Expand Down
53 changes: 32 additions & 21 deletions server/logic/fetchFontSubsetArchive.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ import { config } from "../config";
import { asyncRetry } from "../utils/asyncRetry";
import { IVariantItem } from "./fetchFontURLs";
import { Readable } from "stream";
import axios from "axios";

const RETRIES = 2;
const REQUEST_TIMEOUT_MS = 6000;

export interface IFontSubsetArchive {
zipPath: string; // absolute path to the zip file
Expand Down Expand Up @@ -45,11 +47,11 @@ export async function fetchFontSubsetArchive(
let readable: Readable;
try {
// console.log("fetchFontSubsetArchive...", variantUrl.format, filename, variantUrl.url);
readable = await fetchFontSubsetArchiveStream(variantUrl.url, filename, variantUrl.format);
readable = await fetchFontSubsetArchiveStream(variantUrl.url);
archive.file(filename, readable);
} catch (e) {
// if a specific format does not work, silently discard it.
console.error("fetchFontSubsetArchive discarding", variantUrl.format, filename, variantUrl.url);
console.error("fetchFontSubsetArchive discarding", variantUrl.format, filename, variantUrl.url, e);
return null;
}

Expand Down Expand Up @@ -104,32 +106,41 @@ export async function fetchFontSubsetArchive(
return subsetFontArchive;
}

async function fetchFontSubsetArchiveStream(url: string, dest: string, format: string): Promise<Readable> {
async function fetchFontSubsetArchiveStream(url: string): Promise<Readable> {
return asyncRetry<Readable>(
async () => {
const response = await fetch(url);
const contentType = response.headers.get("content-type");

if (response.status !== 200) {
throw new Error(`${url} fetchFontSubsetArchiveStream request failed. status code: ${response.status} ${response.statusText}`);
}
const res = await axios.get<Readable>(url, {
timeout: REQUEST_TIMEOUT_MS,
responseType: "stream",
maxRedirects: 0 // https://github.com/axios/axios/issues/2610
});

if (_.isNil(contentType) || _.isEmpty(contentType) || contentType.indexOf(format) === -1) {
throw new Error(
`${url} fetchFontSubsetArchiveStream request failed. expected ${format} to be in content-type header: ${contentType}`
);
}
return res.data;

if (_.isNil(response.body)) {
throw new Error(`${url} fetchFontSubsetArchiveStream request failed. response.body is null`);
}
// const response = await fetch(url);
// const contentType = response.headers.get("content-type");

// if (response.status !== 200) {
// throw new Error(`${url} fetchFontSubsetArchiveStream request failed. status code: ${response.status} ${response.statusText}`);
// }

// if (_.isNil(contentType) || _.isEmpty(contentType) || contentType.indexOf(format) === -1) {
// throw new Error(
// `${url} fetchFontSubsetArchiveStream request failed. expected ${format} to be in content-type header: ${contentType}`
// );
// }

// if (_.isNil(response.body)) {
// throw new Error(`${url} fetchFontSubsetArchiveStream request failed. response.body is null`);
// }

// // hold in mem while creating archive.
// return response.arrayBuffer();
// // // hold in mem while creating archive.
// // return response.arrayBuffer();

// TODO typing mismatch ReadableStream<any> vs ReadableStream<Uint8Array>
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return Readable.fromWeb(<any>response.body);
// // TODO typing mismatch ReadableStream<any> vs ReadableStream<Uint8Array>
// // eslint-disable-next-line @typescript-eslint/no-explicit-any
// return Readable.fromWeb(<any>response.body);
},
{ retries: RETRIES }
);
Expand Down
Loading

0 comments on commit 1eaf40a

Please sign in to comment.