From 7200211c699c84cf7e2042305ac371773b6ab731 Mon Sep 17 00:00:00 2001 From: Mario Ranftl Date: Thu, 25 May 2023 21:03:00 +0200 Subject: [PATCH] saner retry bailout, stream again --- server/logic/fetchCSS.ts | 2 +- server/logic/fetchFontSubsetArchive.ts | 54 +++++++++++++------------- server/logic/fetchGoogleFonts.ts | 2 +- server/utils/asyncRetry.ts | 16 +++----- 4 files changed, 35 insertions(+), 39 deletions(-) diff --git a/server/logic/fetchCSS.ts b/server/logic/fetchCSS.ts index 4023882..03e6e7f 100644 --- a/server/logic/fetchCSS.ts +++ b/server/logic/fetchCSS.ts @@ -3,7 +3,7 @@ import * as _ from "lodash"; import { IUserAgents } from "../config"; import { asyncRetry } from "../utils/asyncRetry"; -const RETRIES = 5; +const RETRIES = 2; interface IResource { src: string | null; diff --git a/server/logic/fetchFontSubsetArchive.ts b/server/logic/fetchFontSubsetArchive.ts index baacb45..922b5f9 100644 --- a/server/logic/fetchFontSubsetArchive.ts +++ b/server/logic/fetchFontSubsetArchive.ts @@ -7,8 +7,9 @@ import { finished } from "stream/promises"; import { config } from "../config"; import { asyncRetry } from "../utils/asyncRetry"; import { IVariantItem } from "./fetchFontURLs"; +import { Readable, pipeline } from "stream"; -const RETRIES = 3; +const RETRIES = 2; export interface IFontSubsetArchive { zipPath: string; // absolute path to the zip file @@ -36,19 +37,19 @@ export async function fetchFontSubsetArchive( // const streams: (Readable | fs.WriteStream)[] = _.compact( // _.flatten( - await Bluebird.map(variants, async (variant) => { - return await Bluebird.map(variant.urls, async (variantUrl) => { + await Bluebird.map(variants, (variant) => { + return Bluebird.map(variant.urls, async (variantUrl) => { const filename = `${fontID}-${fontVersion}-${subsets.join("_")}-${variant.id}.${variantUrl.format}`; // download the file for type (filename now known) - let arrayBuffer: ArrayBuffer; + let readable: Readable; try { - console.log("fetchFontSubsetArchive...", fontID, subsets, variantUrl.url, variantUrl.format, filename); - arrayBuffer = await fetchFontSubsetArchiveStream(variantUrl.url, filename, variantUrl.format); - archive.file(filename, arrayBuffer); + // console.log("fetchFontSubsetArchive...", variantUrl.format, filename, variantUrl.url); + readable = await fetchFontSubsetArchiveStream(variantUrl.url, filename, variantUrl.format); + archive.file(filename, readable); } catch (e) { // if a specific format does not work, silently discard it. - console.error("fetchFontSubsetArchive discarding", fontID, subsets, variantUrl.url, variantUrl.format, filename, e); + console.error("fetchFontSubsetArchive discarding", variantUrl.format, filename, variantUrl.url); return null; } @@ -67,21 +68,20 @@ export async function fetchFontSubsetArchive( const target = fs.createWriteStream(subsetFontArchive.zipPath); // streams.push(target); - console.info(`fetchFontSubsetArchive create archive... file=${subsetFontArchive.zipPath}`, fontID, subsets); + console.info(`fetchFontSubsetArchive create archive... file=${subsetFontArchive.zipPath}`); try { - await finished( - archive - .generateNodeStream({ - compression: "DEFLATE", - streamFiles: true, - }) - .pipe(target) - ); - - console.info(`fetchFontSubsetArchive create archive done! file=${subsetFontArchive.zipPath}`, fontID, subsets); + await finished(pipeline(archive.generateNodeStream({ + compression: "DEFLATE", + streamFiles: true, + }), target, (err) => { + if (err) { + console.error("fetchFontSubsetArchive archive.generateNodeStream pipe failed file=${subsetFontArchive.zipPath}", err); + } + })); + console.info(`fetchFontSubsetArchive create archive done! file=${subsetFontArchive.zipPath}`); } catch (e) { - console.error("fetchFontSubsetArchive archive.generateNodeStream pipe failed", fontID, subsets, e); + console.error("fetchFontSubsetArchive archive.generateNodeStream pipe failed", e); // ensure all fs streams into the archive and the actual zip file are destroyed // _.each(streams, (stream) => { // try { @@ -98,8 +98,8 @@ export async function fetchFontSubsetArchive( return subsetFontArchive; } -async function fetchFontSubsetArchiveStream(url: string, dest: string, format: string): Promise { - return asyncRetry( +async function fetchFontSubsetArchiveStream(url: string, dest: string, format: string): Promise { + return asyncRetry( async () => { const response = await fetch(url); const contentType = response.headers.get("content-type"); @@ -118,12 +118,12 @@ async function fetchFontSubsetArchiveStream(url: string, dest: string, format: s throw new Error(`${url} fetchFontSubsetArchiveStream request failed. response.body is null`); } - // hold in mem while creating archive. - return response.arrayBuffer(); + // // hold in mem while creating archive. + // return response.arrayBuffer(); - // // TODO typing mismatch ReadableStream vs ReadableStream - // // eslint-disable-next-line @typescript-eslint/no-explicit-any - // return Readable.fromWeb(response.body); + // TODO typing mismatch ReadableStream vs ReadableStream + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return Readable.fromWeb(response.body); }, { retries: RETRIES } ); diff --git a/server/logic/fetchGoogleFonts.ts b/server/logic/fetchGoogleFonts.ts index 8c833c0..0d9856a 100644 --- a/server/logic/fetchGoogleFonts.ts +++ b/server/logic/fetchGoogleFonts.ts @@ -5,7 +5,7 @@ import * as speakingurl from "speakingurl"; import { config } from "../config"; import { asyncRetry } from "../utils/asyncRetry"; -const RETRIES = 5; +const RETRIES = 2; export interface IFontItem { id: string; diff --git a/server/utils/asyncRetry.ts b/server/utils/asyncRetry.ts index 772fee2..2afeb3a 100644 --- a/server/utils/asyncRetry.ts +++ b/server/utils/asyncRetry.ts @@ -1,13 +1,6 @@ import * as Bluebird from "bluebird"; import * as _ from "lodash"; -// 2 ** 0 * 100 = 100ms -// 2 ** 1 * 100 = 200ms => 300ms -// 2 ** 2 * 100 = 400ms => 700ms -// 2 ** 3 * 100 = 800ms => 1500ms -// 2 ** 4 * 100 = 1600ms => 3100ms -// 2 ** 5 * 100 = 3200ms => 6300ms -// 2 ** 6 * 100 = 6400ms => 12700ms // eslint-disable-next-line @typescript-eslint/no-explicit-any export async function asyncRetry(fn: () => Promise, options: { retries: number }, errors: any[] = []): Promise { let t: T; @@ -21,11 +14,14 @@ export async function asyncRetry(fn: () => Promise, options: { retries: nu ); } - const bailoutMS = 2 ** errors.length * 100; - console.error(`asyncRetry: try ${errors.length + 1} failed, retries=${options.retries}. Delaying next try ${bailoutMS}ms`, e); + // 2 ** 0 * 500 = 500ms + // 2 ** 1 * 500 = 1000ms => 1500ms + // 2 ** 2 * 500 = 2000ms => 3500ms + const bailoutMS = 2 ** errors.length * 500; + // console.error(`asyncRetry: try ${errors.length + 1} failed, retries=${options.retries}. Delaying next try ${bailoutMS}ms`); await Bluebird.delay(bailoutMS); - console.warn(`asyncRetry: retrying after ${bailoutMS}ms`); + // console.warn(`asyncRetry: retrying after ${bailoutMS}ms`); return asyncRetry(fn, options, [...errors, e]); }