feat(fundamentalsTimeSeries): improved financial reports (#753)

* #752!: resolved missing data points.

BREAKING CHANGEs:
- requires option module.
- type is removed from data key.

* Generated schema.

* Fixed prettier warnings.

* Format const with camel case.

* Fixed tests with options module required by schema.

* Saved with LF.

* Support for option module all.

* Aligned modules with Yahoo names and moved keys to json file.

* HAR file script to extract keys from requests.

* Fixed test coverage warnings.

* More time-series keys and minor script improvement.

* Changed keys means changed results :-).

* Test cases per module and period type to verify the results.

.gitignore

@@ -2,3 +2,6 @@
 /coverage
 /dist
 /api
+
+# HAR (HTTP Archive) file format
+*.har

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "files.eol": "\n",
+    "prettier.singleQuote": false,
+    "prettier.quoteProps": "preserve",
+}

docs/modules/fundamentalsTimeSeries.md

@@ -327,7 +327,8 @@ an array of symbols, and you'll receive an array of results back.
 | ------------- | ----------| ---------- | --------------------------------- |
 | `period1`     | Date*     | *required* | Starting period
 | `period2`     | Date*     | (today)    | Ending period
-| `type`        | "quarterly", "annual" | "quarterly" | Financial time series type
+| `type`        | "quarterly", "annual", "trailing" | "quarterly" | Reporting period type
+| `module`        | "financials", "balance-sheet", "cash-flow", "all" | *required* | Financial modules
 | `lang`      | string | `"en-US"` |                                                                   |
 | `region`    | string | `"US"`    |                                                                   |
 

package.json

@@ -39,6 +39,7 @@
     "lint": "eslint . --ext .js,.ts",
     "//schema": "ts-json-schema-generator -f tsconfig.json -p 'src/{modules/**/*.ts,lib/options.ts}' -t '*' | node bin/schema-tweak.js > schema.json",
     "schema": "node --loader ts-node/esm scripts/schema.js > schema.json",
+    "timeseries": "node --loader ts-node/esm scripts/timeseries.js",
     "build": "yarn run build:esm && yarn run build:cjs && yarn run build:post",
     "build:esm": "tsc --module es2020 --target es2019 --outDir dist/esm",
     "build:cjs": "tsc --module commonjs --target es2015 --outDir dist/cjs && sed 's/\"type\": \"module\",/\"type:\": \"commonjs\",/' dist/cjs/package.json > dist/cjs/package-changed.json && mv dist/cjs/package-changed.json dist/cjs/package.json",
@@ -72,6 +73,7 @@
     "@semantic-release/npm": "9.0.2",
     "@semantic-release/release-notes-generator": "10.0.3",
     "@tsconfig/node12": "12.1.1",
+    "@types/har-format": "^1.2.15",
     "@types/jest": "29.5.12",
     "@types/node-fetch": "2.6.11",
     "@typescript-eslint/eslint-plugin": "5.62.0",

schema.json

@@ -1293,10 +1293,14 @@
         },
         "region": {
           "type": "string"
+        },
+        "module": {
+          "type": "string"
         }
       },
       "required": [
-        "period1"
+        "period1",
+        "module"
       ],
       "additionalProperties": false
     },
@@ -1323,10 +1327,25 @@
       ],
       "additionalProperties": false
     },
+    "NamedParameters<typeof processQuery>": {
+      "type": "object",
+      "properties": {
+        "queryOptions": {
+          "$ref": "#/definitions/FundamentalsTimeSeriesOptions",
+          "description": "Input query options."
+        }
+      },
+      "required": [
+        "queryOptions"
+      ],
+      "additionalProperties": false
+    },
     "NamedParameters<typeof processResponse>": {
       "type": "object",
       "properties": {
-        "response": {}
+        "response": {
+          "description": "Query response."
+        }
       },
       "required": [
         "response"

scripts/timeseries.ts

@@ -0,0 +1,95 @@
+/**
+ * Timeseries script extracts keys from requests in a HAR file.
+ * It reads the HAR file as input to update the timeseries.json.
+ * A HAR file can be saved from the Network tab (see Developer tools).
+ * Filter the results on the time-series API url to create the HAR file.
+ * See: INPUT_PATH for the location.
+ */
+import fs from "fs/promises";
+import type { PathLike } from "fs";
+import type { Har } from "har-format";
+
+const INPUT_PATH = "timeseries.har";
+const OUTPUT_PATH = "timeseries.json";
+const STRIP_REGEX = new RegExp("annual|quarterly|trailing");
+
+async function fileExists(path: PathLike): Promise<boolean> {
+  return await fs
+    .access(INPUT_PATH)
+    .then(() => true)
+    .catch(() => {
+      console.log(`File not found: ${path}`);
+      process.exit(1);
+    });
+}
+
+/* Read the HAR file. */
+async function getTimeseriesHAR(): Promise<Har> {
+  await fileExists(INPUT_PATH);
+  const fileBuffer = await fs.readFile(INPUT_PATH, "utf8");
+  return JSON.parse(fileBuffer) as Har;
+}
+
+/* Read the timeseries JSON file. */
+async function getTimeseriesJSON(): Promise<Record<string, string[]>> {
+  await fileExists(OUTPUT_PATH);
+  const fileBuffer = await fs.readFile(OUTPUT_PATH, "utf8");
+  return JSON.parse(fileBuffer);
+}
+
+/* Main async script function. */
+async function main() {
+  const har = await getTimeseriesHAR();
+
+  for (const entry of har.log.entries) {
+    const json = await getTimeseriesJSON();
+    const refererHeader = entry.request.headers.find(
+      (header) => header.name === "Referer"
+    );
+    const typeParam = entry.request.queryString.find(
+      (header) => header.name === "type"
+    );
+
+    if (!refererHeader) {
+      throw new Error("Referer header not found.");
+    } else if (!typeParam) {
+      throw new Error("Type query parameter not found.");
+    } else {
+      console.log(`Entry: ${entry.startedDateTime}: ${refererHeader.value}`);
+    }
+
+    /* Extract the type of request from the referer. */
+    const module = refererHeader.value.split("/").pop();
+    const keysArray = module ? json[module] : undefined;
+
+    /* Use the first key to confirm the module. */
+    if (keysArray && !typeParam.value.includes(keysArray[0])) {
+      console.log(`${module} does not contain key: ${keysArray[0]}`);
+      console.log("Referer most likely does not contain the right module.");
+      continue;
+    } else if (keysArray && module) {
+      const keySet = new Set(keysArray);
+      const paramKeys = typeParam.value.split(",");
+      console.log(`Module: ${module} with ${keySet.size} keys.`);
+      console.log(`Query param includes ${paramKeys.length} keys.`);
+
+      for (const key of paramKeys) {
+        const stripped = key.replace(STRIP_REGEX, "");
+        keySet.add(stripped);
+      }
+      json[module] = Array.from(keySet);
+
+      try {
+        await fs.writeFile(OUTPUT_PATH, JSON.stringify(json, null, 2));
+      } catch (err: any) {
+        console.error(err);
+        process.exit(1);
+      } finally {
+        const counter = keySet.size - keysArray.length;
+        console.log(`Updated ${module} with ${counter} new keys.`);
+      }
+    } else throw new Error(`Keys array not found for module: ${module}`);
+  }
+}
+
+main();