feat(yahoo): Configurable YF_QUERY_HOST in URLs (#109)

We no longer hard code e.g. `query1.finance.yahoo.com` in modules, instead using `${YF_QUERY_HOST}` which may be set via `setGlobalConfig` or an environment variable. * Standardize all relevant module URLs * Use `query2` for all tests (and http cache) * Variable substitution code
gadicc · Mar 6, 2022 · 716c0f1 · 716c0f1
1 parent 7bc89e0
commit 716c0f1
Show file tree

Hide file tree

Showing 132 changed files with 202 additions and 204 deletions.
diff --git a/docs/other/setGlobalConfig.md b/docs/other/setGlobalConfig.md
@@ -17,8 +17,8 @@ yahooFinance.setGlobalConfig({
 Notes:
 
 - Config provided to this function is validated.
-
 - Options are merged infinite levels deep:
+
 ```js
 import yahooFinance from 'yahoo-finance2';
 
@@ -29,3 +29,18 @@ yahooFinance.setGlobalConfig({
     }
 });
 ```
+
+## Options not documented elsewhere
+
+* `YF_QUERY_HOST`
+
+  * Default: `query2.yahoo.finance.com`
+  * Description: the host to use to query Yahoo's API.
+  As per
+  [this stackoverflow post](https://stackoverflow.com/questions/44030983/yahoo-finance-url-not-working/47505102#47505102):
+
+    * `query1.finance.yahoo.com` serves `HTTP/1.0`
+    * `query2.finance.yahoo.com` serves `HTTP/1.1`
+    * [Differences between HTTP/1.0 and HTTP/1.1](https://stackoverflow.com/questions/246859/http-1-0-vs-1-1)
+
+  * This option in particular may also be set with an environment variable of the same name.
diff --git a/schema.json b/schema.json
@@ -5,6 +5,9 @@
     "YahooFinanceOptions": {
       "type": "object",
       "properties": {
+        "YF_QUERY_HOST": {
+          "type": "string"
+        },
         "queue": {
           "$ref": "#/definitions/QueueOptions"
         },

diff --git a/src/lib/fetchDevel.js b/src/lib/fetchDevel.js
@@ -33,6 +33,12 @@ async function fetchDevel(url, fetchOptions) {
   if (process.env.FETCH_DEVEL === "nocache")
     return await nodeFetch(url, fetchOptions);
 
+  // Use query2 for all our tests / fixtures / cache
+  url = url.replace(
+    /^https:\/\/query1.finance.yahoo.com/,
+    "https://query2.finance.yahoo.com"
+  );
+
   // If devel===true, hash the url, otherwise use the value of devel
   // This allows us to specify our own static filename vs url hash.
   /*

diff --git a/src/lib/options.ts b/src/lib/options.ts
@@ -3,11 +3,13 @@ import type { ValidationOptions } from "./validateAndCoerceTypes.js";
 import type { QueueOptions } from "./queue.js";
 
 export interface YahooFinanceOptions {
+  YF_QUERY_HOST?: string;
   queue?: QueueOptions;
   validation?: ValidationOptions;
 }
 
 const options: YahooFinanceOptions = {
+  YF_QUERY_HOST: process.env.YF_QUERY_HOST || "query2.finance.yahoo.com",
   queue: {
     concurrency: 4, // Min: 1, Max: Infinity
     timeout: 60,

diff --git a/src/lib/yahooFinanceFetch.spec.ts b/src/lib/yahooFinanceFetch.spec.ts
@@ -2,7 +2,9 @@ import * as util from "util";
 import { jest } from "@jest/globals";
 
 import Queue from "./queue.js";
-import _yahooFinanceFetch from "./yahooFinanceFetch.js";
+import _yahooFinanceFetch, {
+  substituteVariables,
+} from "./yahooFinanceFetch.js";
 import errors from "./errors.js";
 
 import _env from "../env-node.js";
@@ -44,7 +46,7 @@ describe("yahooFinanceFetch", () => {
     it("throws HTTPError if !res.ok and no error in json result", () => {
       return expect(
         yahooFinanceFetch(
-          "https://query1.finance.yahoo.com/nonExistingURL-CACHED",
+          "https://query2.finance.yahoo.com/nonExistingURL-CACHED",
           {},
           { devel: "pageWith404andJson.fake.json" }
         )
@@ -54,7 +56,7 @@ describe("yahooFinanceFetch", () => {
   it("throws Error if we receive unknown error from json result", () => {
     return expect(
       yahooFinanceFetch(
-        "https://query1.finance.yahoo.com/nonExistingURL-CACHED",
+        "https://query2.finance.yahoo.com/nonExistingURL-CACHED",
         {},
         { devel: "pageWithUnknownError.json" }
       )
@@ -170,4 +172,31 @@ describe("yahooFinanceFetch", () => {
 
     // TODO, timeout test
   });
+
+  describe("URL variable substitution", () => {
+    const that = { _opts };
+
+    it("subs YF_QUERY_HOST from _opts", () => {
+      const origUrl = "https://${YF_QUERY_HOST}/v8/something";
+      // @ts-ignore: partial This for testing
+      const newUrl = substituteVariables.call(that, origUrl);
+      expect(newUrl).toBe("https://query2.finance.yahoo.com/v8/something");
+    });
+
+    it("subs query2 if no option", () => {
+      const that = { _env, _opts: { ..._opts } };
+      delete that._opts.YF_QUERY_HOST;
+      const origUrl = "https://${YF_QUERY_HOST}/v8/something";
+      // @ts-ignore: partial This for testing
+      const newUrl = substituteVariables.call(that, origUrl);
+      expect(newUrl).toBe("https://query2.finance.yahoo.com/v8/something");
+    });
+
+    it("leaves non-matches", () => {
+      const origUrl = "${something}";
+      // @ts-ignore: partial This for testing
+      const newUrl = substituteVariables.call(that, origUrl);
+      expect(newUrl).toBe("${something}");
+    });
+  });
 });
diff --git a/src/lib/yahooFinanceFetch.ts b/src/lib/yahooFinanceFetch.ts
@@ -41,6 +41,19 @@ function assertQueueOptions(queue: any, opts: any) {
     queue.timeout = opts.timeout;
 }
 
+function substituteVariables(this: YahooFinanceFetchThis, urlBase: string) {
+  return urlBase.replace(/\$\{([^\}]+)\}/g, (match, varName) => {
+    if (varName === "YF_QUERY_HOST") {
+      // const hosts = ["query1.finance.yahoo.com", "query2.finance.yahoo.com"];
+      // return hosts[Math.floor(Math.random() * hosts.length)];
+      return this._opts.YF_QUERY_HOST || "query2.finance.yahoo.com";
+    } else {
+      // i.e. return unsubstituted original variable expression ${VAR}
+      return match;
+    }
+  });
+}
+
 async function yahooFinanceFetch(
   this: YahooFinanceFetchThis,
   urlBase: string,
@@ -62,7 +75,8 @@ async function yahooFinanceFetch(
 
   // @ts-ignore TODO copy interface? @types lib?
   const urlSearchParams = new URLSearchParams(params);
-  const url = urlBase + "?" + urlSearchParams.toString();
+  const url =
+    substituteVariables.call(this, urlBase) + "?" + urlSearchParams.toString();
 
   /* istanbul ignore next */
   // no need to force coverage on real network request.
@@ -114,4 +128,5 @@ async function yahooFinanceFetch(
   return result;
 }
 
+export { substituteVariables };
 export default yahooFinanceFetch;
diff --git a/src/modules/chart.ts b/src/modules/chart.ts
@@ -205,7 +205,7 @@ export default async function _chart(
 
     query: {
       assertSymbol: symbol,
-      url: "https://query1.finance.yahoo.com/v8/finance/chart/" + symbol,
+      url: "https://${YF_QUERY_HOST}/v8/finance/chart/" + symbol,
       schemaKey: "#/definitions/ChartOptions",
       defaults: queryOptionsDefaults,
       overrides: queryOptionsOverrides,

diff --git a/src/modules/historical.ts b/src/modules/historical.ts
@@ -57,7 +57,7 @@ export default function historical(
 
     query: {
       assertSymbol: symbol,
-      url: "https://query1.finance.yahoo.com/v7/finance/download/" + symbol,
+      url: "https://${YF_QUERY_HOST}/v7/finance/download/" + symbol,
       schemaKey: "#/definitions/HistoricalOptions",
       defaults: queryOptionsDefaults,
       overrides: queryOptionsOverrides,

diff --git a/src/modules/insights.ts b/src/modules/insights.ts
@@ -157,7 +157,7 @@ export default function trendingSymbols(
     moduleName: "insights",
     query: {
       assertSymbol: symbol,
-      url: "https://query2.finance.yahoo.com/ws/insights/v2/finance/insights",
+      url: "https://${YF_QUERY_HOST}/ws/insights/v2/finance/insights",
       schemaKey: "#/definitions/InsightsOptions",
       defaults: queryOptionsDefaults,
       overrides: queryOptionsOverrides,

diff --git a/src/modules/options.ts b/src/modules/options.ts
@@ -82,7 +82,7 @@ export default function options(
 
     query: {
       assertSymbol: symbol,
-      url: "https://query1.finance.yahoo.com/v7/finance/options/" + symbol,
+      url: "https://${YF_QUERY_HOST}/v7/finance/options/" + symbol,
       schemaKey: "#/definitions/OptionsOptions",
       defaults: queryOptionsDefaults,
       overrides: queryOptionsOverrides,

diff --git a/src/modules/quote.ts b/src/modules/quote.ts
@@ -247,7 +247,7 @@ export default async function quote(
     moduleName: "quote",
 
     query: {
-      url: "https://query2.finance.yahoo.com/v7/finance/quote",
+      url: "https://${YF_QUERY_HOST}/v7/finance/quote",
       schemaKey: "#/definitions/QuoteOptions",
       defaults: queryOptionsDefaults,
       runtime: { symbols },

diff --git a/src/modules/quoteSummary.ts b/src/modules/quoteSummary.ts
@@ -115,8 +115,7 @@ export default function quoteSummary(
 
     query: {
       assertSymbol: symbol,
-      url:
-        "https://query2.finance.yahoo.com/v10/finance/quoteSummary/" + symbol,
+      url: "https://${YF_QUERY_HOST}/v10/finance/quoteSummary/" + symbol,
       schemaKey: "#/definitions/QuoteSummaryOptions",
       defaults: queryOptionsDefaults,
       overrides: queryOptionsOverrides,

diff --git a/src/modules/recommendationsBySymbol.ts b/src/modules/recommendationsBySymbol.ts
@@ -55,7 +55,9 @@ export default function recommendationsBySymbol(
     moduleName: "recommendationsBySymbol",
 
     query: {
-      url: `https://query2.finance.yahoo.com/v6/finance/recommendationsbysymbol/${symbols}`,
+      url:
+        "https://${YF_QUERY_HOST}/v6/finance/recommendationsbysymbol/" +
+        symbols,
       schemaKey: "#/definitions/RecommendationsBySymbolOptions",
       defaults: queryOptionsDefaults,
       overrides: queryOptionsOverrides,

diff --git a/src/modules/search.ts b/src/modules/search.ts
@@ -168,7 +168,7 @@ export default function search(
     moduleName: "search",
 
     query: {
-      url: "https://query2.finance.yahoo.com/v1/finance/search",
+      url: "https://${YF_QUERY_HOST}/v1/finance/search",
       schemaKey: "#/definitions/SearchOptions",
       defaults: queryOptionsDefaults,
       runtime: { q: query },

diff --git a/src/modules/trendingSymbols.ts b/src/modules/trendingSymbols.ts
@@ -52,7 +52,7 @@ export default function trendingSymbols(
   return this._moduleExec({
     moduleName: "trendingSymbols",
     query: {
-      url: `https://query1.finance.yahoo.com/v1/finance/trending/${query}`,
+      url: "https://${YF_QUERY_HOST}/v1/finance/trending/" + query,
       schemaKey: "#/definitions/TrendingSymbolsOptions",
       defaults: queryOptionsDefaults,
       overrides: queryOptionsOverrides,

diff --git a/tests/http/4-7-2021/historical-BEKE-2020-01-01-to-2020-01-03.json b/tests/http/4-7-2021/historical-BEKE-2020-01-01-to-2020-01-03.json
@@ -1,6 +1,6 @@
 {
   "request": {
-    "url": "https://query1.finance.yahoo.com/v7/finance/download/BEKE?interval=1d&events=history&includeAdjustedClose=true&period1=1577836800&period2=1578009600"
+    "url": "https://query2.finance.yahoo.com/v7/finance/download/BEKE?interval=1d&events=history&includeAdjustedClose=true&period1=1577836800&period2=1578009600"
   },
   "response": {
     "ok": false,

diff --git a/tests/http/4-7-2021/historical-BFLY-2020-01-01-to-2020-01-03.json b/tests/http/4-7-2021/historical-BFLY-2020-01-01-to-2020-01-03.json
@@ -1,6 +1,6 @@
 {
   "request": {
-    "url": "https://query1.finance.yahoo.com/v7/finance/download/BFLY?interval=1d&events=history&includeAdjustedClose=true&period1=1577836800&period2=1578009600"
+    "url": "https://query2.finance.yahoo.com/v7/finance/download/BFLY?interval=1d&events=history&includeAdjustedClose=true&period1=1577836800&period2=1578009600"
   },
   "response": {
     "ok": false,

diff --git a/tests/http/4-7-2021/historical-BRKS-2020-01-01-to-2020-01-03.json b/tests/http/4-7-2021/historical-BRKS-2020-01-01-to-2020-01-03.json
@@ -1,6 +1,6 @@
 {
   "request": {
-    "url": "https://query1.finance.yahoo.com/v7/finance/download/BRKS?interval=1d&events=history&includeAdjustedClose=true&period1=1577836800&period2=1578009600"
+    "url": "https://query2.finance.yahoo.com/v7/finance/download/BRKS?interval=1d&events=history&includeAdjustedClose=true&period1=1577836800&period2=1578009600"
   },
   "response": {
     "ok": true,

diff --git a/tests/http/4-7-2021/historical-CRON-2020-01-01-to-2020-01-03.json b/tests/http/4-7-2021/historical-CRON-2020-01-01-to-2020-01-03.json
diff --git a/tests/http/4-7-2021/historical-SIMP-2020-01-01-to-2020-01-03.json b/tests/http/4-7-2021/historical-SIMP-2020-01-01-to-2020-01-03.json
@@ -1,6 +1,6 @@
 {
   "request": {
-    "url": "https://query1.finance.yahoo.com/v7/finance/download/SIMP?interval=1d&events=history&includeAdjustedClose=true&period1=1577836800&period2=1578009600"
+    "url": "https://query2.finance.yahoo.com/v7/finance/download/SIMP?interval=1d&events=history&includeAdjustedClose=true&period1=1577836800&period2=1578009600"
   },
   "response": {
     "ok": false,

diff --git a/tests/http/4-7-2021/trendingSymbols-BRKS.json b/tests/http/4-7-2021/trendingSymbols-BRKS.json
@@ -1,6 +1,6 @@
 {
   "request": {
-    "url": "https://query1.finance.yahoo.com/v7/finance/options/BRKS?formatted=false&lang=en-US&region=US"
+    "url": "https://query2.finance.yahoo.com/v7/finance/options/BRKS?formatted=false&lang=en-US&region=US"
   },
   "response": {
     "ok": true,

diff --git a/tests/http/chart-0P000071W8.TO-2020-01-01-to-2020-01-03.json b/tests/http/chart-0P000071W8.TO-2020-01-01-to-2020-01-03.json
@@ -1,6 +1,6 @@
 {
   "request": {
-    "url": "https://query1.finance.yahoo.com/v8/finance/chart/0P000071W8.TO?useYfid=true&interval=1d&includePrePost=true&events=div%7Csplit%7Cearn&lang=en-US&period1=1577836800&period2=1578009600"
+    "url": "https://query2.finance.yahoo.com/v8/finance/chart/0P000071W8.TO?useYfid=true&interval=1d&includePrePost=true&events=div%7Csplit%7Cearn&lang=en-US&period1=1577836800&period2=1578009600"
   },
   "response": {
     "ok": true,

diff --git a/tests/http/chart-AAPL-2020-01-01-to-2020-01-03.json b/tests/http/chart-AAPL-2020-01-01-to-2020-01-03.json
@@ -1,6 +1,6 @@
 {
   "request": {
-    "url": "https://query1.finance.yahoo.com/v8/finance/chart/AAPL?useYfid=true&interval=1d&includePrePost=true&events=div%7Csplit%7Cearn&lang=en-US&period1=1577836800&period2=1578009600"
+    "url": "https://query2.finance.yahoo.com/v8/finance/chart/AAPL?useYfid=true&interval=1d&includePrePost=true&events=div%7Csplit%7Cearn&lang=en-US&period1=1577836800&period2=1578009600"
   },
   "response": {
     "ok": true,

diff --git a/tests/http/chart-AAPL-2020-08-31-to-2020-11-07.json b/tests/http/chart-AAPL-2020-08-31-to-2020-11-07.json
@@ -1,6 +1,6 @@
 {
   "request": {
-    "url": "https://query1.finance.yahoo.com/v8/finance/chart/AAPL?useYfid=true&interval=1d&includePrePost=true&events=div%7Csplit%7Cearn&lang=en-US&period1=1598832000&period2=1604707200"
+    "url": "https://query2.finance.yahoo.com/v8/finance/chart/AAPL?useYfid=true&interval=1d&includePrePost=true&events=div%7Csplit%7Cearn&lang=en-US&period1=1598832000&period2=1604707200"
   },
   "response": {
     "ok": true,

diff --git a/tests/http/chart-AAPL.json b/tests/http/chart-AAPL.json
@@ -1,6 +1,6 @@
 {
   "request": {
-    "url": "https://query1.finance.yahoo.com/v8/finance/chart/AAPL?useYfid=true&interval=1d&includePrePost=true&events=div%7Csplit%7Cearn&lang=en-US&period1=1645488000&period2=1645574400"
+    "url": "https://query2.finance.yahoo.com/v8/finance/chart/AAPL?useYfid=true&interval=1d&includePrePost=true&events=div%7Csplit%7Cearn&lang=en-US&period1=1645488000&period2=1645574400"
   },
   "response": {
     "ok": true,