saner retry bailout, stream again

majodev · May 25, 2023 · 7200211 · 7200211
1 parent b99380b
commit 7200211
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 39 deletions.
diff --git a/server/logic/fetchCSS.ts b/server/logic/fetchCSS.ts
@@ -3,7 +3,7 @@ import * as _ from "lodash";
 import { IUserAgents } from "../config";
 import { asyncRetry } from "../utils/asyncRetry";
 
-const RETRIES = 5;
+const RETRIES = 2;
 
 interface IResource {
   src: string | null;

diff --git a/server/logic/fetchFontSubsetArchive.ts b/server/logic/fetchFontSubsetArchive.ts
@@ -7,8 +7,9 @@ import { finished } from "stream/promises";
 import { config } from "../config";
 import { asyncRetry } from "../utils/asyncRetry";
 import { IVariantItem } from "./fetchFontURLs";
+import { Readable, pipeline } from "stream";
 
-const RETRIES = 3;
+const RETRIES = 2;
 
 export interface IFontSubsetArchive {
   zipPath: string; // absolute path to the zip file
@@ -36,19 +37,19 @@ export async function fetchFontSubsetArchive(
 
   // const streams: (Readable | fs.WriteStream)[] = _.compact(
   //   _.flatten(
-  await Bluebird.map(variants, async (variant) => {
-    return await Bluebird.map(variant.urls, async (variantUrl) => {
+  await Bluebird.map(variants, (variant) => {
+    return Bluebird.map(variant.urls, async (variantUrl) => {
       const filename = `${fontID}-${fontVersion}-${subsets.join("_")}-${variant.id}.${variantUrl.format}`;
 
       // download the file for type (filename now known)
-      let arrayBuffer: ArrayBuffer;
+      let readable: Readable;
       try {
-        console.log("fetchFontSubsetArchive...", fontID, subsets, variantUrl.url, variantUrl.format, filename);
-        arrayBuffer = await fetchFontSubsetArchiveStream(variantUrl.url, filename, variantUrl.format);
-        archive.file(filename, arrayBuffer);
+        // console.log("fetchFontSubsetArchive...", variantUrl.format, filename, variantUrl.url);
+        readable = await fetchFontSubsetArchiveStream(variantUrl.url, filename, variantUrl.format);
+        archive.file(filename, readable);
       } catch (e) {
         // if a specific format does not work, silently discard it.
-        console.error("fetchFontSubsetArchive discarding", fontID, subsets, variantUrl.url, variantUrl.format, filename, e);
+        console.error("fetchFontSubsetArchive discarding", variantUrl.format, filename, variantUrl.url);
         return null;
       }
 
@@ -67,21 +68,20 @@ export async function fetchFontSubsetArchive(
   const target = fs.createWriteStream(subsetFontArchive.zipPath);
   // streams.push(target);
 
-  console.info(`fetchFontSubsetArchive create archive... file=${subsetFontArchive.zipPath}`, fontID, subsets);
+  console.info(`fetchFontSubsetArchive create archive... file=${subsetFontArchive.zipPath}`);
 
   try {
-    await finished(
-      archive
-        .generateNodeStream({
-          compression: "DEFLATE",
-          streamFiles: true,
-        })
-        .pipe(target)
-    );
-
-    console.info(`fetchFontSubsetArchive create archive done! file=${subsetFontArchive.zipPath}`, fontID, subsets);
+    await finished(pipeline(archive.generateNodeStream({
+      compression: "DEFLATE",
+      streamFiles: true,
+    }), target, (err) => {
+      if (err) {
+        console.error("fetchFontSubsetArchive archive.generateNodeStream pipe failed file=${subsetFontArchive.zipPath}", err);
+      }
+    }));
+    console.info(`fetchFontSubsetArchive create archive done! file=${subsetFontArchive.zipPath}`);
   } catch (e) {
-    console.error("fetchFontSubsetArchive archive.generateNodeStream pipe failed", fontID, subsets, e);
+    console.error("fetchFontSubsetArchive archive.generateNodeStream pipe failed", e);
     // ensure all fs streams into the archive and the actual zip file are destroyed
     // _.each(streams, (stream) => {
     //   try {
@@ -98,8 +98,8 @@ export async function fetchFontSubsetArchive(
   return subsetFontArchive;
 }
 
-async function fetchFontSubsetArchiveStream(url: string, dest: string, format: string): Promise<ArrayBuffer> {
-  return asyncRetry<ArrayBuffer>(
+async function fetchFontSubsetArchiveStream(url: string, dest: string, format: string): Promise<Readable> {
+  return asyncRetry<Readable>(
     async () => {
       const response = await fetch(url);
       const contentType = response.headers.get("content-type");
@@ -118,12 +118,12 @@ async function fetchFontSubsetArchiveStream(url: string, dest: string, format: s
         throw new Error(`${url} fetchFontSubsetArchiveStream request failed. response.body is null`);
       }
 
-      // hold in mem while creating archive.
-      return response.arrayBuffer();
+      // // hold in mem while creating archive.
+      // return response.arrayBuffer();
 
-      // // TODO typing mismatch ReadableStream<any> vs ReadableStream<Uint8Array>
-      // // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      // return Readable.fromWeb(<any>response.body);
+      // TODO typing mismatch ReadableStream<any> vs ReadableStream<Uint8Array>
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      return Readable.fromWeb(<any>response.body);
     },
     { retries: RETRIES }
   );

diff --git a/server/logic/fetchGoogleFonts.ts b/server/logic/fetchGoogleFonts.ts
@@ -5,7 +5,7 @@ import * as speakingurl from "speakingurl";
 import { config } from "../config";
 import { asyncRetry } from "../utils/asyncRetry";
 
-const RETRIES = 5;
+const RETRIES = 2;
 
 export interface IFontItem {
   id: string;

diff --git a/server/utils/asyncRetry.ts b/server/utils/asyncRetry.ts
@@ -1,13 +1,6 @@
 import * as Bluebird from "bluebird";
 import * as _ from "lodash";
 
-// 2 ** 0 * 100 = 100ms
-// 2 ** 1 * 100 = 200ms  => 300ms
-// 2 ** 2 * 100 = 400ms  => 700ms
-// 2 ** 3 * 100 = 800ms  => 1500ms
-// 2 ** 4 * 100 = 1600ms => 3100ms
-// 2 ** 5 * 100 = 3200ms => 6300ms
-// 2 ** 6 * 100 = 6400ms => 12700ms
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export async function asyncRetry<T>(fn: () => Promise<T>, options: { retries: number }, errors: any[] = []): Promise<T> {
   let t: T;
@@ -21,11 +14,14 @@ export async function asyncRetry<T>(fn: () => Promise<T>, options: { retries: nu
       );
     }
 
-    const bailoutMS = 2 ** errors.length * 100;
-    console.error(`asyncRetry: try ${errors.length + 1} failed, retries=${options.retries}. Delaying next try ${bailoutMS}ms`, e);
+    // 2 ** 0 * 500 = 500ms
+    // 2 ** 1 * 500 = 1000ms  => 1500ms
+    // 2 ** 2 * 500 = 2000ms  => 3500ms
+    const bailoutMS = 2 ** errors.length * 500;
+    // console.error(`asyncRetry: try ${errors.length + 1} failed, retries=${options.retries}. Delaying next try ${bailoutMS}ms`);
     await Bluebird.delay(bailoutMS);
 
-    console.warn(`asyncRetry: retrying after ${bailoutMS}ms`);
+    // console.warn(`asyncRetry: retrying after ${bailoutMS}ms`);
     return asyncRetry(fn, options, [...errors, e]);
   }