add coin price data fetcher

script/coin-price-data-fetcher/config/default.js

@@ -0,0 +1,35 @@
+module.exports  = {
+  sourceCurrency: 'ADA',
+  targetFiatCurrencies: ['USD', 'JPY', 'EUR', 'CNY', 'KRW'],
+  targetCryptoCurrencies: ['ETH', 'BTC'],
+  logger: {
+    level: 'info'
+  },
+  exchangeRateRefreshInterval: 10*60*1000,
+  serviceDataFreshnessThreshold: 2*60*1000,
+  // monitor allows 10% difference between the price from the service and directly from the API
+  monitorDiscrepancyThreshold: 0.1,
+  fetchTimeout: 30*1000,
+  apiKeys: {
+    cryptocompare: process.env.API_KEY_CRYPTOCOMPARE,
+    coinlayer: process.env.API_KEY_COINLAYER,
+    coinmarketcap: process.env.API_KEY_COINMARKETCAP,
+    coinapi: process.env.API_KEY_COINAPI,
+    nomics: process.env.API_KEY_NOMICS,
+    cryptoapis: process.env.API_KEY_CRYPTOAPIS,
+    openexchangerates: process.env.API_KEY_OPENEXCHANGERATES,
+  },
+
+  privKeyData: process.env.COIN_PRICE_PRIV_KEY,
+  pubKeyData: process.env.COIN_PRICE_PUB_KEY,
+
+  s3: {
+    region: process.env.PRICE_DATA_S3_REGION,
+    bucketName: process.env.PRICE_DATA_S3_BUCKET_NAME,
+    accessKeyId: process.env.PRICE_DATA_S3_ACCESS_KEY_ID,
+    secretAccessKey: process.env.PRICE_DATA_S3_SECRET_ACCESS_KEY,
+  },
+
+  exchangeRateCachePath: '/tmp/exchange-rates.json',
+  exchangeRateCacheTime: 60*60*1000, // 1 hour
+}

script/coin-price-data-fetcher/config/develop.js

@@ -0,0 +1,26 @@
+module.exports = {
+  fetcherProviders: [
+    //'cryptocompare',
+    //'coinlayer',
+    //'coinmarketcap',
+    //'coinapi',
+    //'coinpaprika',
+    //'nomics',
+    'cryptonator',
+    //'shrimpy',
+    //'cryptoapis',
+    // 'badMockApi',
+  ],
+  monitorProviders: [
+    // 'cryptocompare',
+    // 'coinlayer',
+    // 'coinmarketcap',
+    // 'coinapi',
+    // 'coinpaprika',
+    // 'nomics',
+    // 'cryptonator',
+    'shrimpy',
+    // 'cryptoapis',
+    // 'badMockApi',
+  ],
+};

script/coin-price-data-fetcher/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "coin-price-data-fetcher",
+  "version": "0.0.1",
+  "description": "coin price data fetcher",
+  "main": "index.js",
+  "scripts": {
+    "flow": "flow",
+    "postinstall": "npm run flow-remove-types",
+    "flow-remove-types": "flow-remove-types -d ./flow-files/config --all --pretty config/ && flow-remove-types -d ./flow-files/src --all --pretty src/",
+    "start-fetcher": "NODE_ENV=${NODE_ENV:-develop} node ./flow-files/src/index.js",
+    "start-monitor": "NODE_ENV=${NODE_ENV:-develop} node ./flow-files/src/index.js monitor",
+    "test": "npm run flow-remove-types && jest ./flow-files/src"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@emurgo/cardano-serialization-lib-nodejs": "^9.1.4",
+    "aws-sdk": "^2.1057.0",
+    "bunyan": "^1.8.12",
+    "config": "^3.2.2",
+    "fetch-timeout": "0.0.2",
+    "flow": "^0.2.3",
+    "flow-remove-types": "^2.106.2",
+    "node-fetch": "^2.6.0"
+  },
+  "devDependencies": {
+    "jest": "^24.9.0"
+  },
+  "jest": {
+    "testEnvironment": "node"
+  }
+}

script/coin-price-data-fetcher/src/api.js

@@ -0,0 +1,142 @@
+// @flow
+// This module defines the API providers.
+import type { PairRate } from './types';
+
+class ErrorResponse extends Error {
+  constructor(msg?: string) {
+    super(msg || 'Unexpected resposne');
+  }
+}
+
+export type FetchFunc = (url: string, headers?: { [header:string]: string }) =>
+  Promise<Object>;
+export type ApiFunc = (fetch: FetchFunc, apiKey: string) => Promise<Array<PairRate>>;
+
+// Provides ADA-all
+const cryptocompare: ApiFunc = async (fetch, apiKey) => {
+  const response = await fetch(`https://min-api.cryptocompare.com/data/price?fsym=ADA&tsyms=USD,JPY,EUR,CNY,KRW,BTC,ETH&api_key=${apiKey}`);
+  if (response.Response === 'Error') {
+    throw new ErrorResponse();
+  }
+  return ['USD', 'JPY', 'EUR', 'CNY', 'KRW', 'BTC', 'ETH'].map(to =>
+    ({ from: 'ADA', to, price: response[to] }));
+};
+
+// Provides ADA-USD, BTC-USD, ETH-USD
+const coinlayer: ApiFunc = async (fetch, apiKey) => {
+  const response = await fetch(`http://api.coinlayer.com/api/live?access_key=${apiKey}&symbols=ADA,BTC,ETH&target=USD`);
+  if (response.success !== true) {
+    throw new ErrorResponse();
+  }
+  return Object.keys(response.rates).map(from => ({ from, to: 'USD', price: response.rates[from] }));
+};
+
+// Provides ADA-USD, BTC-USD, ETH-USD
+const coinmarketcap: ApiFunc = async (fetch, apiKey) => {
+  const response = await fetch(`https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest?symbol=ADA,ETH,BTC&convert=USD`, {
+    'X-CMC_PRO_API_KEY': apiKey,
+    'Accept': 'application/json'
+  });
+  if (response.status.error_code !== 0) {
+    throw new ErrorResponse();
+  }
+  return ['ETH', 'ADA', 'BTC'].map(from =>
+    ({ from, to: 'USD', price: response.data[from].quote.USD.price }));
+};
+
+// Provides ADA-USD, ADA-BTC, ADA-ETH
+const coinapi: ApiFunc = async (fetch, apiKey) => {
+  return Promise.all(['USD', 'BTC', 'ETH'].map(async (to) => {
+    const response = await fetch(`https://rest.coinapi.io/v1/exchangerate/ADA/${to}`,
+      { 'X-CoinAPI-Key': apiKey });
+    return { from: 'ADA', to, price: response.rate };
+  }));
+};
+
+// Provides ADA-all
+const coinpaprika: ApiFunc = async (fetch, _apiKey) => {
+  const response = await fetch('https://api.coinpaprika.com/v1/tickers/ada-cardano?quotes=USD,BTC,ETH,JPY,EUR,CNY,KRW');
+  return ['USD', 'JPY', 'EUR', 'CNY', 'KRW', 'BTC', 'ETH'].map(to =>
+    ({ from: 'ADA', to, price: response.quotes[to].price }));
+};
+
+// Provides ADA-USD,BTC,ETH
+const nomics: ApiFunc = async (fetch, apiKey) => {
+  let result;
+  // fetch ETH,BTC to ADA rate and take the reciprocals
+  let response = await fetch(`https://api.nomics.com/v1/currencies/ticker?key=${apiKey}&ids=ETH,BTC&convert=ADA&interval=1h`);
+  result = response.map(data => 
+    ({ from: 'ADA', to: data.symbol, price: 1/Number(data.price) })
+  );
+
+  // fetch ADA-USD
+  response = await fetch(`https://api.nomics.com/v1/currencies/ticker?key=${apiKey}&ids=ADA&convert=USD&interval=1h`);
+  result.push({ from: 'ADA', to: 'USD', price: Number(response[0].price) });
+
+  return result;
+};
+
+// Provides ADA-all except CNY
+const cryptonator: ApiFunc = async (fetch, apiKey) => {
+  // cryptonator doesn't have CNY or KRW
+  return Promise.all(['usd', 'jpy', 'eur', 'btc', 'eth'].map(async (to) => {
+    const response = await fetch(`https://api.cryptonator.com/api/ticker/ada-${to}`);
+    if (response.success !== true) {
+      throw new ErrorResponse();
+    }
+    return { from: 'ADA', to: response.ticker.target, price: Number(response.ticker.price) };
+  }));
+};
+
+// Provides ADA-USD,BTC,ETH
+const shrimpy: ApiFunc = async (fetch, _apiKey) => {
+  const response = await fetch('https://dev-api.shrimpy.io/v1/exchanges/kraken/ticker');
+  const adaData = response.find(data => data.symbol === 'ADA');
+  const ethData = response.find(data => data.symbol === 'ETH');
+
+  return [
+    { from: 'ADA', to: 'USD', price: Number(adaData.priceUsd) },
+    { from: 'ADA', to: 'BTC', price: Number(adaData.priceBtc) },
+    { from: 'ADA', to: 'ETH', price: Number(adaData.priceUsd) / Number(ethData.priceUsd) }
+  ];      
+};
+
+// Provides ADA-BTC,ETH,USD,EUR
+const cryptoapis: ApiFunc = async (fetch, apiKey) => {
+  /* source:
+     curl -H 'X-API-Key:b67bf0950e65a556579e2d27e1c3914d44158628' 'https://api.cryptoapis.io/v1/assets/meta?skip=skip&limit=limit' | python -mjson.tool */
+  const ids = {
+    ADA: '5b1ea92e584bf5002013062d',
+    BTC: '5b1ea92e584bf50020130612',
+    USD: '5b1ea92e584bf50020130615',
+    EUR: '5b1ea92e584bf5002013061a',
+    ETH: '5b755dacd5dd99000b3d92b2',
+  }
+  return Promise.all(['BTC', 'ETH', 'USD'].map(async (to) => {
+    const response = await fetch(`https://api.cryptoapis.io/v1/exchange-rates/${ids.ADA}/${ids[to]}`, 
+      { 'X-API-Key': apiKey });
+    return { from: 'ADA', to, price: response.payload.weightedAveragePrice }
+  }));
+};
+
+// A mock API that always fails, for testing
+const badMockApi: ApiFunc = async (_fetch, _apiKey) => {
+  throw new Error('bad mock API fails');
+};
+
+module.exports = {
+  ErrorResponse,
+  providers: {
+    cryptocompare,
+    coinlayer,
+    coinmarketcap,
+    coinapi,
+    coinpaprika,
+    nomics,
+    cryptonator,
+    shrimpy,
+    cryptoapis,
+    badMockApi,
+  },
+};
+

script/coin-price-data-fetcher/src/fetcher.js

@@ -0,0 +1,115 @@
+// @flow
+// This modules fetches data using APIs defined in api.js.
+const config = require('config');
+const fetch = require('fetch-timeout');
+const api = require('./api');
+const logger = require('./logger');
+const exchangeRate = require('./fiat-exchange-rate');
+
+import type { PairRate } from './types';
+
+async function queryApi(apiName: string): Promise<Array<PairRate>> {
+  let retry = 0;
+
+  async function _fetch(url, headers) {
+    let __logger = logger.child({ apiName, url, retry });
+    __logger.info('fetching ' + url, headers);
+    const response = await fetch(url, { headers }, config.fetchTimeout, 'fetch timeout');
+    if (!response.ok) {
+      __logger.error('error', response.status, response.statusText); 
+      throw new Error('Fetch error');
+    }
+    const json = await response.json();
+    __logger.info('response', json);
+    return json;
+  }
+
+  for (;;) {
+    const _logger = logger.child({ api: apiName, retry });
+
+    try {
+      const result = await api.providers[apiName](_fetch, config.apiKeys[apiName]);
+      _logger.info(`got data from ${apiName}`, { result });
+      return result;
+    } catch (error) {
+      _logger.error(error);
+      if (retry === 2) {
+        throw new Error(`failed to get data from ${apiName}`)
+      }
+      retry++;
+    }
+  } //for
+  throw new Error('will never reach here');
+}
+
+/* 
+  Each API provider has different capabilities. Some lacks exchange rates 
+  against certain fiat currencies. This function calculates missing exchanges so
+  that the returned ticker set include pairs of ADA and all target currencies. 
+*/
+function normalizeQueryResult(queryResult: Array<PairRate>): Array<PairRate> {
+  function findPair(from, to) {
+    return queryResult.find(pair => (pair.from === from) && (pair.to === to));
+  }
+  const result = [];
+  const adaUsd = findPair('ADA', 'USD');
+  if (!adaUsd) {
+    throw new Error('missing ADA-USD rate');
+  }
+  result.push(adaUsd);
+
+  for (const fiat of config.targetFiatCurrencies.filter(s => s !== 'USD')) {
+    const pair = findPair('ADA', fiat);
+    if (pair) {
+      result.push(pair);
+    } else {
+      const price = adaUsd.price * exchangeRate.getFromUsd(fiat);
+      result.push({ from: 'ADA', to: fiat, price });
+    }
+  }
+
+  for (const crypto of config.targetCryptoCurrencies) {
+    const pair = findPair('ADA', crypto);
+    if (pair) {
+      result.push(pair);
+      continue;
+    } 
+
+    const cryptoUsd = findPair(crypto, 'USD');
+    if (!cryptoUsd) {
+      throw new Error(`missing ${crypto} rate`);
+    }
+    const price = adaUsd.price/cryptoUsd.price;
+    result.push({ from: 'ADA', to: crypto, price });
+  }
+
+  return result;
+}
+
+async function queryAndNormalize(apiName: string): Promise<?Array<PairRate>> {
+  let apiResult;
+  try {
+    apiResult = await queryApi(apiName);
+  } catch (_) {
+    return null;
+  }
+  return normalizeQueryResult(apiResult);
+}
+
+module.exports = { queryAndNormalize };
+
+//NODE_ENV=development node fetcher.js | ./node_modules/.bin/bunyan -c 'this.msg.match(/got data from/)'
+if (require.main === module) {
+  (async () => {
+    await exchangeRate.start();
+    for (const apiName in api.providers) {
+      console.log('-'.repeat(30)+apiName+'-'.repeat(30));
+      const result = await queryApi(apiName);
+      logger.info('normalized result', normalizeQueryResult(result));
+    }
+    process.exit(0);
+  })().catch(error => {
+    console.error(error);
+  });
+}
+