feat(getCrumb): ability to specify custom cookie jar / store (#670)

src/lib/cookieJar.ts

@@ -1,6 +1,6 @@
 import { Cookie, CookieJar } from "tough-cookie";
 
-export class MyCookieJar extends CookieJar {
+export class ExtendedCookieJar extends CookieJar {
   async setFromSetCookieHeaders(
     setCookieHeader: string | Array<string>,
     url: string
@@ -24,7 +24,3 @@ export class MyCookieJar extends CookieJar {
         }
   }
 }
-
-const cookiejar = new MyCookieJar();
-
-export default cookiejar;

src/lib/getCrumb.spec.ts

@@ -1,25 +1,30 @@
 import env from "../env-node";
 import getCrumb, { _getCrumb, getCrumbClear } from "./getCrumb";
-import { MyCookieJar } from "./cookieJar.js";
 import { jest } from "@jest/globals";
 import { consoleSilent, consoleRestore } from "../../tests/console.js";
 
+import { ExtendedCookieJar } from "./cookieJar.js";
+
 describe("getCrumb", () => {
-  beforeAll(consoleSilent);
+  let cookieJar: ExtendedCookieJar;
+  beforeAll(() => {
+    consoleSilent();
+    cookieJar = new ExtendedCookieJar();
+  });
   afterAll(consoleRestore);
 
   describe("_getCrumb", () => {
     it("finds crumb in context", async () => {
       const fetch = await env.fetchDevel();
 
       const crumb = await _getCrumb(
+        new ExtendedCookieJar(),
         fetch,
         // @ts-expect-error: fetchDevel still has no types (yet)
         { devel: true },
         "https://finance.yahoo.com/quote/AAPL",
         "getCrumb-quote-AAPL.json",
-        true,
-        new MyCookieJar()
+        true
       );
       expect(crumb).toBe("mloUP8q7ZPH");
     });
@@ -28,6 +33,7 @@ describe("getCrumb", () => {
       const fetch = await env.fetchDevel();
 
       const crumb = await _getCrumb(
+        cookieJar,
         fetch,
         // @ts-expect-error: fetchDevel still has no types (yet)
         { devel: true }
@@ -39,6 +45,7 @@ describe("getCrumb", () => {
       const fetch = await env.fetchDevel();
 
       let crumb = await _getCrumb(
+        cookieJar,
         fetch,
         // @ts-expect-error: fetchDevel still has no types (yet)
         { devel: true },
@@ -49,6 +56,7 @@ describe("getCrumb", () => {
       // TODO, at tests to see how many times fetch was called, etc.
 
       crumb = await _getCrumb(
+        cookieJar,
         fetch,
         // @ts-expect-error: fetchDevel still has no types (yet)
         { devel: true },
@@ -62,13 +70,13 @@ describe("getCrumb", () => {
 
       await expect(() =>
         _getCrumb(
+          new ExtendedCookieJar(),
           fetch,
           // @ts-expect-error: fetchDevel still has no types (yet)
           { devel: true },
           "https://finance.yahoo.com/quote/AAPL",
           "getCrumb-quote-AAPL-no-cookies.fake.json",
-          true,
-          new MyCookieJar()
+          true
         )
       ).rejects.toThrowError(/No set-cookie/);
     });
@@ -78,13 +86,13 @@ describe("getCrumb", () => {
 
       await expect(() =>
         _getCrumb(
+          new ExtendedCookieJar(),
           fetch,
           // @ts-expect-error: fetchDevel still has no types (yet)
           { devel: true },
           "https://finance.yahoo.com/quote/AAPL",
           "getCrumb-quote-AAPL-no-context.fake.json",
-          true,
-          new MyCookieJar()
+          true
         )
       ).rejects.toThrowError(/Could not find window.YAHOO.context/);
     });
@@ -94,13 +102,13 @@ describe("getCrumb", () => {
 
       await expect(() =>
         _getCrumb(
+          new ExtendedCookieJar(),
           fetch,
           // @ts-expect-error: fetchDevel still has no types (yet)
           { devel: true },
           "https://finance.yahoo.com/quote/AAPL",
           "getCrumb-quote-AAPL-invalid-json.fake.json",
-          true,
-          new MyCookieJar()
+          true
         )
       ).rejects.toThrowError(/Could not parse window.YAHOO.context/);
     });
@@ -110,13 +118,13 @@ describe("getCrumb", () => {
 
       await expect(() =>
         _getCrumb(
+          new ExtendedCookieJar(),
           fetch,
           // @ts-expect-error: fetchDevel still has no types (yet)
           { devel: true },
           "https://finance.yahoo.com/quote/AAPL",
           "getCrumb-quote-AAPL-no-crumb.fake.json",
-          true,
-          new MyCookieJar()
+          true
         )
       ).rejects.toThrowError(/Could not find crumb/);
     });
@@ -126,13 +134,13 @@ describe("getCrumb", () => {
       const fetch = await env.fetchDevel();
 
       const crumb = await _getCrumb(
+        new ExtendedCookieJar(),
         fetch,
         // @ts-expect-error: fetchDevel still has no types (yet)
         { devel: true },
         "https://finance.yahoo.com/quote/AAPL",
         "getCrumb-quote-AAPL-pre-consent-VPN-UK.json",
-        true,
-        new MyCookieJar()
+        true
       );
       expect(crumb).toBe("Ky3Po5TGQRZ");
       // consoleSilent();
@@ -141,9 +149,10 @@ describe("getCrumb", () => {
 
   describe("getCrumb", () => {
     it("works", async () => {
-      await getCrumbClear();
+      await getCrumbClear(cookieJar);
       const fetch = await env.fetchDevel();
       const crumb = await getCrumb(
+        cookieJar,
         fetch,
         // @ts-expect-error: fetchDevel still has no types (yet)
         { devel: true }
@@ -154,9 +163,10 @@ describe("getCrumb", () => {
     it("only calls getCrumb once", async () => {
       const fetch = await env.fetchDevel();
       const _getCrumb = jest.fn(() => "crumb");
-      await getCrumbClear();
+      await getCrumbClear(cookieJar);
 
       getCrumb(
+        cookieJar,
         fetch,
         // @ts-expect-error: fetchDevel still has no types (yet)
         { devel: true },
@@ -165,6 +175,7 @@ describe("getCrumb", () => {
       );
 
       getCrumb(
+        cookieJar,
         fetch,
         // @ts-expect-error: fetchDevel still has no types (yet)
         { devel: true },

src/lib/getCrumb.ts

@@ -1,5 +1,5 @@
 import type { RequestInfo, RequestInit, Response } from "node-fetch";
-import defaultCookieJar from "./cookieJar.js";
+import type { ExtendedCookieJar } from "./cookieJar";
 
 let crumb: string | null = null;
 // let crumbFetchTime = 0;
@@ -11,12 +11,12 @@ const parseHtmlEntities = (str: string) =>
   );
 
 export async function _getCrumb(
+  cookieJar: ExtendedCookieJar,
   fetch: (url: RequestInfo, init?: RequestInit) => Promise<Response>,
   fetchOptionsBase: RequestInit,
   url = "https://finance.yahoo.com/quote/AAPL",
   develOverride = "getCrumb-quote-AAPL.json",
-  noCache = false,
-  cookieJar = defaultCookieJar
+  noCache = false
 ): Promise<string | null> {
   // if (crumb && crumbFetchTime + MAX_CRUMB_CACHE_TIME > Date.now()) return crumb;
 
@@ -207,12 +207,12 @@ export async function _getCrumb(
         */
 
         return await _getCrumb(
+          cookieJar,
           fetch,
           finalResponseFetchOptions,
           copyConsentResponseLocation,
           "getCrumb-quote-AAPL-consent-final-redirect.html",
-          noCache,
-          cookieJar
+          noCache
         );
       }
     } else {
@@ -272,14 +272,15 @@ export async function _getCrumb(
 let promise: Promise<string | null> | null = null;
 let promiseTime = 0;
 
-export async function getCrumbClear() {
+export async function getCrumbClear(cookieJar: ExtendedCookieJar) {
   crumb = null;
   promise = null;
   promiseTime = 0;
-  await defaultCookieJar.removeAllCookies();
+  await cookieJar.removeAllCookies();
 }
 
 export default function getCrumb(
+  cookieJar: ExtendedCookieJar,
   fetch: (url: RequestInfo, init?: RequestInit) => Promise<Response>,
   fetchOptionsBase: RequestInit,
   url = "https://finance.yahoo.com/quote/AAPL",
@@ -288,7 +289,7 @@ export default function getCrumb(
   // TODO, rather do this with cookie expire time somehow
   const now = Date.now();
   if (!promise || now - promiseTime > 60_000) {
-    promise = __getCrumb(fetch, fetchOptionsBase, url);
+    promise = __getCrumb(cookieJar, fetch, fetchOptionsBase, url);
     promiseTime = now;
   }
 

src/lib/options.ts

@@ -1,15 +1,18 @@
 // TODO, keep defaults there too?
 import type { ValidationOptions } from "./validateAndCoerceTypes.js";
 import type { QueueOptions } from "./queue.js";
+import { ExtendedCookieJar } from "./cookieJar.js";
 
 export interface YahooFinanceOptions {
   YF_QUERY_HOST?: string;
+  cookieJar: ExtendedCookieJar;
   queue?: QueueOptions;
   validation?: ValidationOptions;
 }
 
 const options: YahooFinanceOptions = {
   YF_QUERY_HOST: process.env.YF_QUERY_HOST || "query2.finance.yahoo.com",
+  cookieJar: new ExtendedCookieJar(),
   queue: {
     concurrency: 4, // Min: 1, Max: Infinity
     timeout: 60,

src/lib/setGlobalConfig.ts

@@ -13,6 +13,7 @@ export default function setGlobalConfig(
     options: this._opts.validation,
     schemaKey: "#/definitions/YahooFinanceOptions",
   });
+  // @ts-expect-error: TODO
   mergeObjects(this._opts, config as Obj);
 }
 

src/lib/yahooFinanceFetch.spec.ts

@@ -120,38 +120,42 @@ describe("yahooFinanceFetch", () => {
       expect(queue.concurrency).toBe(5);
     });
 
-    it("yahooFinanceFetch branch check for alternate queue", () => {
+    it("yahooFinanceFetch branch check for alternate queue", async () => {
       const promises = [
         yahooFinanceFetch("", {}),
         yahooFinanceFetch("", {}, {}),
         yahooFinanceFetch("", {}, { queue: {} }),
       ];
 
+      await immediate();
+
       env.fetch.fetches[0].resolveWith({ ok: true });
       env.fetch.fetches[1].resolveWith({ ok: true });
       env.fetch.fetches[2].resolveWith({ ok: true });
 
-      return Promise.all(promises);
+      await Promise.all(promises);
     });
 
-    it("assert defualts to {} for empty queue opts", () => {
+    it("assert defualts to {} for empty queue opts", async () => {
       moduleOpts.queue.concurrency = 1;
       const opts = { ..._opts };
       // @ts-ignore: intentional to test runtime failures
       delete opts.queue;
       const yahooFinanceFetch = _yahooFinanceFetch.bind({ _env: env, _opts });
 
       const promise = yahooFinanceFetch("", {}, moduleOpts);
+      await immediate();
       env.fetch.fetches[0].resolveWith({ ok: true });
-      return expect(promise).resolves.toMatchObject({ ok: true });
+      await expect(promise).resolves.toMatchObject({ ok: true });
     });
 
-    it("single item in queue", () => {
+    it("single item in queue", async () => {
       moduleOpts.queue.concurrency = 1;
 
       const promise = yahooFinanceFetch("", {}, moduleOpts);
+      await immediate();
       env.fetch.fetches[0].resolveWith({ ok: true });
-      return expect(promise).resolves.toMatchObject({ ok: true });
+      await expect(promise).resolves.toMatchObject({ ok: true });
     });
 
     it("waits if exceeding concurrency max", async () => {
@@ -161,6 +165,7 @@ describe("yahooFinanceFetch", () => {
         yahooFinanceFetch("", {}, moduleOpts),
         yahooFinanceFetch("", {}, moduleOpts),
       ];
+      await immediate();
 
       // Second func should not be called until 1st reoslves (limit 1)
       expect(env.fetch.fetches.length).toBe(1);

src/lib/yahooFinanceFetch.ts

@@ -7,7 +7,6 @@ import type { QueueOptions } from "./queue.js";
 import errors from "./errors.js";
 import pkg from "../../package.json";
 import getCrumb from "./getCrumb.js";
-import cookieJar from "./cookieJar.js";
 
 const userAgent = `${pkg.name}/${pkg.version} (+${pkg.repository})`;
 
@@ -62,7 +61,7 @@ function substituteVariables(this: YahooFinanceFetchThis, urlBase: string) {
 async function yahooFinanceFetch(
   this: YahooFinanceFetchThis,
   urlBase: string,
-  params = {},
+  params: Record<string, string> = {},
   moduleOpts: YahooFinanceFetchModuleOptions = {},
   func = "json",
   needsCrumb = false
@@ -93,8 +92,12 @@ async function yahooFinanceFetch(
   };
 
   if (needsCrumb) {
-    // @ts-expect-error: TODO, crumb string type for partial params
-    params.crumb = await getCrumb(fetchFunc, fetchOptionsBase);
+    const crumb = await getCrumb(
+      this._opts.cookieJar,
+      fetchFunc,
+      fetchOptionsBase
+    );
+    if (crumb) params.crumb = crumb;
   }
 
   // @ts-expect-error: TODO copy interface? @types lib?
@@ -109,7 +112,9 @@ async function yahooFinanceFetch(
     ...fetchOptionsBase,
     headers: {
       ...fetchOptionsBase.headers,
-      cookie: cookieJar.getCookieStringSync(url, { allPaths: true }),
+      cookie: await this._opts.cookieJar.getCookieString(url, {
+        allPaths: true,
+      }),
     },
   };
 
@@ -122,7 +127,7 @@ async function yahooFinanceFetch(
 
   const setCookieHeaders = response.headers.raw()["set-cookie"];
   if (setCookieHeaders)
-    cookieJar.setFromSetCookieHeaders(setCookieHeaders, url);
+    this._opts.cookieJar.setFromSetCookieHeaders(setCookieHeaders, url);
 
   const result = await response[func]();