Merge pull request #159 from blockfrost/fix/partial-exchange-rates

fix: return partial exchange rates, throttle requests

package.json

@@ -24,6 +24,7 @@
     "express": "^4.17.1",
     "got": "^11.8.3",
     "memoizee": "^0.4.15",
+    "rate-limiter-flexible": "^2.3.6",
     "serialize-error": "^8.1.0",
     "uuid": "^8.3.2",
     "validator": "^13.6.0",

src/constants/config.ts

@@ -1,4 +1,15 @@
 // BIP44 gap limit https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki#address-gap-limit
 export const ADDRESS_GAP_LIMIT = 20;
+
 // Skip emitting of `newBlock` event for missed blocks if we missed more than defined number of them
 export const EMIT_MAX_MISSED_BLOCKS = 3;
+
+// List of coingecko-compatible proxies to retrieve historical fiat rates for balance history endpoint
+// Set env variable BLOCKFROST_FIAT_RATES_PROXY to provide additional proxies (comma separated values)
+// eg. BLOCKFROST_FIAT_RATES_PROXY="https://example.com/api/v3/coins/cardano/history,https://example2.com/history"
+export const FIAT_RATES_PROXY = ['https://api.coingecko.com/api/v3/coins/cardano/history'];
+
+// Max number of requests per second sent to FIAT_RATES_PROXY, additional requests will be queued
+export const FIAT_RATES_REQUESTS_PER_SEC = 100;
+// Request timeout for fetching single rate for a given day
+export const FIAT_RATES_REQUESTS_TIMEOUT = 1000;

src/methods/getBalanceHistory.ts

@@ -4,7 +4,7 @@ import { BalanceHistoryData } from '../types/response';
 import { TxIdsToTransactionsResponse } from '../types/transactions';
 import { getAccountTransactionIds } from '../utils/account';
 import { sumAssetBalances } from '../utils/asset';
-import { getRatesForDate } from '../utils/common';
+import { getRatesForDate } from '../utils/rates';
 import { prepareErrorMessage, prepareMessage } from '../utils/message';
 import { txIdsToTransactions } from '../utils/transaction';
 
@@ -129,12 +129,15 @@ export const getAccountBalanceHistory = async (
 
   // fetch fiat rate for each bin
   const binRatesPromises = bins.map(bin => getRatesForDate(bin.time));
-  const binRates = await Promise.all(binRatesPromises);
+  const binRates = await Promise.allSettled(binRatesPromises);
 
-  const result = bins.map((bin, index) => ({
-    ...bin,
-    rates: binRates[index],
-  }));
+  const result = bins.map((bin, index) => {
+    const rateForBin = binRates[index];
+    return {
+      ...bin,
+      rates: rateForBin.status === 'fulfilled' ? rateForBin.value : {},
+    };
+  });
 
   return result;
 };

src/types/response.ts

@@ -69,5 +69,5 @@ export interface BalanceHistoryData {
   received: string;
   sent: string;
   sentToSelf: string;
-  rates: Record<string, number>;
+  rates: { [k: string]: number | undefined };
 }

src/types/transactions.ts

@@ -19,15 +19,6 @@ export interface TxIdsToTransactionsPromises {
   promise: Promise<Data>;
 }
 
-export interface BalanceHistoryItem {
-  time: number;
-  txs: number;
-  received: string;
-  sent: string;
-  sentToSelf: string;
-  rates: Record<string, number>;
-}
-
 export interface TransformedTransactionUtxo {
   /** Transaction hash */
   hash: string;

src/utils/common.ts

@@ -1,6 +1,4 @@
-import { format } from 'date-fns';
 import { blockfrostAPI } from '../utils/blockfrostAPI';
-import got from 'got';
 import { NetworkInfo } from '@emurgo/cardano-serialization-lib-nodejs';
 
 export const paginate = <T>(items: T[], pageSize: number): T[][] => {
@@ -12,31 +10,6 @@ export const paginate = <T>(items: T[], pageSize: number): T[][] => {
   }, [] as T[][]);
 };
 
-const formatCoingeckoTime = (date: number): string => {
-  return format(date * 1000, 'dd-MM-yyyy');
-};
-
-export const getRatesForDate = async (date: number): Promise<Record<string, number>> => {
-  const coingeckoDateFormat = formatCoingeckoTime(date);
-  try {
-    const response: {
-      market_data?: {
-        current_price: Record<string, number>;
-      };
-    } = await got(
-      `https://api.coingecko.com/api/v3/coins/cardano/history?date=${coingeckoDateFormat}`,
-    ).json();
-
-    if (!response?.market_data) {
-      throw Error(`Failed to fetch exchange rate for ${coingeckoDateFormat}`);
-    }
-
-    return response.market_data?.current_price;
-  } catch (error) {
-    throw Error(`Failed to fetch exchange rate for ${coingeckoDateFormat}`);
-  }
-};
-
 export const getNetworkId = (): number => {
   const networkId = blockfrostAPI.options.isTestnet
     ? NetworkInfo.testnet().network_id()

src/utils/message.ts

@@ -3,7 +3,7 @@ import { serializeError } from 'serialize-error';
 import { Messages } from '../types/message';
 import * as TxTypes from '../types/transactions';
 import { UtxosWithBlockResponse } from '../types/address';
-import { AccountInfo, ServerInfo } from '../types/response';
+import { AccountInfo, BalanceHistoryData, ServerInfo } from '../types/response';
 
 export const getMessage = (message: string): Messages | null => {
   try {
@@ -45,7 +45,7 @@ export const prepareMessage = (
     | AccountInfo
     | string
     | Responses['block_content']
-    | TxTypes.BalanceHistoryItem[]
+    | BalanceHistoryData[]
     | TxTypes.TxIdsToTransactionsResponse[]
     | TxTypes.TransformedTransaction
     | UtxosWithBlockResponse[]

src/utils/rates.ts

@@ -0,0 +1,89 @@
+import { format } from 'date-fns';
+import got from 'got';
+import {
+  FIAT_RATES_REQUESTS_PER_SEC,
+  FIAT_RATES_PROXY,
+  FIAT_RATES_REQUESTS_TIMEOUT,
+} from '../constants/config';
+import { RateLimiterMemory, RateLimiterQueue } from 'rate-limiter-flexible';
+import { blockfrostAPI } from './blockfrostAPI';
+
+// limit max number of requests per sec to prevent too many opened connections
+const ratesLimiter = new RateLimiterMemory({
+  points: FIAT_RATES_REQUESTS_PER_SEC,
+  duration: 1,
+});
+
+const limiterQueue = new RateLimiterQueue(ratesLimiter);
+
+export const formatCoingeckoTime = (date: number): string => {
+  return format(date * 1000, 'dd-MM-yyyy');
+};
+
+export const getFiatRatesProxies = (additional = process.env.BLOCKFROST_FIAT_RATES_PROXY) => {
+  let proxies = FIAT_RATES_PROXY;
+  if (additional) {
+    const items = additional.split(',');
+    const sanitized = items
+      .map(item => (item.endsWith('/') ? item.substring(0, item.length - 1) : item))
+      .filter(proxy => proxy.length > 0); // remove trailing slash
+    proxies = sanitized.concat(proxies);
+  }
+  return proxies;
+};
+
+export const getRatesForDateNoLimit = async (date: number): Promise<Record<string, number>> => {
+  const coingeckoDateFormat = formatCoingeckoTime(date);
+  try {
+    let response: {
+      market_data?: {
+        current_price: Record<string, number>;
+      };
+    } = {};
+
+    for (const [index, proxy] of getFiatRatesProxies().entries()) {
+      // iterate through proxies till we have a valid response
+      try {
+        response = await got(`${proxy}?date=${coingeckoDateFormat}`, {
+          headers: {
+            'User-Agent': blockfrostAPI.userAgent,
+          },
+          timeout: {
+            request: FIAT_RATES_REQUESTS_TIMEOUT,
+          },
+        }).json();
+        if (response?.market_data?.current_price) {
+          break;
+        }
+      } catch (err) {
+        if (index === FIAT_RATES_PROXY.length - 1) {
+          // last proxy thrown error, we don't have the data
+          throw err;
+        }
+      }
+    }
+
+    if (!response?.market_data) {
+      throw Error(`Failed to fetch exchange rate for ${coingeckoDateFormat}`);
+    }
+
+    return response.market_data?.current_price;
+  } catch (error) {
+    throw Error(`Failed to fetch exchange rate for ${coingeckoDateFormat}`);
+  }
+};
+
+export const getRatesForDate = async (date: number): Promise<Record<string, number>> => {
+  const t1 = new Date().getTime();
+
+  // wait for a slot
+  await limiterQueue.removeTokens(1);
+
+  const t2 = new Date().getTime();
+  const diff = t2 - t1;
+  if (diff > 1000) {
+    console.warn(`Fiat rates limiter slowed down request for ${diff} ms!`);
+  }
+
+  return getRatesForDateNoLimit(date);
+};

test/unit/fixtures/rates.ts

@@ -0,0 +1,42 @@
+import { FIAT_RATES_PROXY } from '../../../src/constants/config';
+
+export const getFiatRatesProxies = [
+  {
+    description: 'getFiatRatesProxies: basic',
+    additional: '',
+    result: [...FIAT_RATES_PROXY],
+  },
+  {
+    description: 'getFiatRatesProxies: passing additional proxy',
+    additional: 'https://mynextproxy.com/history',
+    result: ['https://mynextproxy.com/history', ...FIAT_RATES_PROXY],
+  },
+  {
+    description: 'getFiatRatesProxies: passing additional proxy with trailing slash',
+    additional: 'https://mynextproxy.com/history/',
+    result: ['https://mynextproxy.com/history', ...FIAT_RATES_PROXY],
+  },
+  {
+    description: 'getFiatRatesProxies: passing additional proxies separated by comma',
+    additional: 'https://mynextproxy.com/history,mybiggestproxy.com',
+    result: ['https://mynextproxy.com/history', 'mybiggestproxy.com', ...FIAT_RATES_PROXY],
+  },
+  {
+    description: 'getFiatRatesProxies: passing additional proxies  with trailing comma',
+    additional: 'https://mynextproxy.com/history,mybiggestproxy.com,',
+    result: ['https://mynextproxy.com/history', 'mybiggestproxy.com', ...FIAT_RATES_PROXY],
+  },
+];
+
+export const formatCoingeckoTime = [
+  {
+    description: 'formatCoingeckoTime: basic',
+    time: 1641561868,
+    result: '07-01-2022',
+  },
+  {
+    description: 'formatCoingeckoTime: basic 2',
+    time: 1643545468,
+    result: '30-01-2022',
+  },
+];

test/unit/tests/utils/rates.test.ts

@@ -0,0 +1,16 @@
+import * as ratesUtils from '../../../../src/utils/rates';
+import * as fixtures from '../../fixtures/rates';
+
+describe('asset utils', () => {
+  fixtures.getFiatRatesProxies.forEach(fixture => {
+    test(fixture.description, () => {
+      expect(ratesUtils.getFiatRatesProxies(fixture.additional)).toMatchObject(fixture.result);
+    });
+  });
+
+  fixtures.formatCoingeckoTime.forEach(fixture => {
+    test(fixture.description, () => {
+      expect(ratesUtils.formatCoingeckoTime(fixture.time)).toBe(fixture.result);
+    });
+  });
+});

yarn-project.nix

@@ -601,6 +601,7 @@ let
     { name = "queue-microtask-1.2.3"; filename = "queue-microtask-npm-1.2.3-fcc98e4e2d-b676f8c040.zip"; sha512 = "b676f8c040cdc5b12723ad2f91414d267605b26419d5c821ff03befa817ddd10e238d22b25d604920340fd73efd8ba795465a0377c4adf45a4a41e4234e42dc4"; locatorHash = "fcc98e4e2dd552fad14dec4b1e9ddc6cc30e483993cb69194dcc724ae65bcdb9e9d1f43a687954ac59b9a940570a5850ff5f79b680b6d4e57d9c8849e5b8661b"; }
     { name = "quick-lru-5.1.1"; filename = "quick-lru-npm-5.1.1-e38e0edce3-a516faa255.zip"; sha512 = "a516faa25574be7947969883e6068dbe4aa19e8ef8e8e0fd96cddd6d36485e9106d85c0041a27153286b0770b381328f4072aa40d3b18a19f5f7d2b78b94b5ed"; locatorHash = "e38e0edce3433d36eb8d04780ae8391ea36d0ba87fdf815410cd72ddddf8f5b7ba0b2c4d4c3d835a23e41868c13db2b3985ec57e1d08e49ec83b0958537c1c90"; }
     { name = "range-parser-1.2.1"; filename = "range-parser-npm-1.2.1-1a470fa390-0a268d4fea.zip"; sha512 = "0a268d4fea508661cf5743dfe3d5f47ce214fd6b7dec1de0da4d669dd4ef3d2144468ebe4179049eff253d9d27e719c88dae55be64f954e80135a0cada804ec9"; locatorHash = "1a470fa390fba391a81b3f6a1a394c1a61110bc204810a2f43330d6e2bf02362679bae60670b5ef1cdbf537f6bfd331993043cd8827af67714037ce522ef8812"; }
+    { name = "rate-limiter-flexible-2.3.6"; filename = "rate-limiter-flexible-npm-2.3.6-07a4dd65e8-c008c7bcc1.zip"; sha512 = "c008c7bcc17832645d4c06caa08a5f52e093108d4f3bcca5df15344150a38fca6b823a6718091a8cf887861989c61fd91d7179426d277d7867fb76b723fd0dbc"; locatorHash = "07a4dd65e87ec003b0127b2c93bc8a7daafdf57fd1aac27a503e6f0bd911a76a9a45d7c9e8b6fd4a4493a90f4f6b23bbc636998416b1a4adfdb14889246fe3c1"; }
     { name = "raw-body-2.4.0"; filename = "raw-body-npm-2.4.0-14d9d633af-6343906939.zip"; sha512 = "6343906939e018c6e633a34a938a5d6d1e93ffcfa48646e00207d53b418e941953b521473950c079347220944dc75ba10e7b3c08bf97e3ac72c7624882db09bb"; locatorHash = "14d9d633af8905cdcf9735507751302acac7f9e78dfa62a6aa8b519259f503eec9a55894922dfe33c7f1870181d3f15e0a3a559318cc34ea55388e2071b2b686"; }
     { name = "react-is-17.0.2"; filename = "react-is-npm-17.0.2-091bbb8db6-9d6d111d89.zip"; sha512 = "9d6d111d8990dc98bc5402c1266a808b0459b5d54830bbea24c12d908b536df7883f268a7868cfaedde3dd9d4e0d574db456f84d2e6df9c4526f99bb4b5344d8"; locatorHash = "091bbb8db6f816221d53fd22c86fdeee32df7c5b7db185933240203011e13048c912d9fef515a52196454914cc9496641324fbcf321847041f22f84b430cc89a"; }
     { name = "readable-stream-2.3.7"; filename = "readable-stream-npm-2.3.7-77b22a9818-e4920cf754.zip"; sha512 = "e4920cf7549a60f8aaf694d483a0e61b2a878b969d224f89b3bc788b8d920075132c4b55a7494ee944c7b6a9a0eada28a7f6220d80b0312ece70bbf08eeca755"; locatorHash = "77b22a9818e50f183171e517ad08311291419a450449e8ef50a4e4884d24c8af894c7388093836f89d4271eceeda719e8eb97ecfa6926226bea1fd86a93d5f89"; }

yarn.lock

@@ -739,6 +739,7 @@ __metadata:
     memoizee: ^0.4.15
     nock: ^13.2.1
     prettier: ^2.4.1
+    rate-limiter-flexible: ^2.3.6
     rimraf: ^3.0.2
     serialize-error: ^8.1.0
     sinon: ^11.1.2
@@ -5897,6 +5898,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"rate-limiter-flexible@npm:^2.3.6":
+  version: 2.3.6
+  resolution: "rate-limiter-flexible@npm:2.3.6"
+  checksum: c008c7bcc17832645d4c06caa08a5f52e093108d4f3bcca5df15344150a38fca6b823a6718091a8cf887861989c61fd91d7179426d277d7867fb76b723fd0dbc
+  languageName: node
+  linkType: hard
+
 "raw-body@npm:2.4.0":
   version: 2.4.0
   resolution: "raw-body@npm:2.4.0"