From be4e6d41fd297a576ac13a6e7370f5109c0551bc Mon Sep 17 00:00:00 2001 From: yushi Date: Thu, 13 Jan 2022 14:13:14 +0800 Subject: [PATCH] add coin price data fetcher --- .../coin-price-data-fetcher/config/default.js | 35 +++++ .../coin-price-data-fetcher/config/develop.js | 26 ++++ script/coin-price-data-fetcher/package.json | 32 ++++ script/coin-price-data-fetcher/src/api.js | 142 ++++++++++++++++++ script/coin-price-data-fetcher/src/fetcher.js | 115 ++++++++++++++ .../src/fetcher.test.js | 49 ++++++ .../src/fiat-exchange-rate.js | 108 +++++++++++++ script/coin-price-data-fetcher/src/index.js | 61 ++++++++ script/coin-price-data-fetcher/src/logger.js | 6 + script/coin-price-data-fetcher/src/monitor.js | 58 +++++++ script/coin-price-data-fetcher/src/sign.js | 49 ++++++ script/coin-price-data-fetcher/src/types.js | 13 ++ .../coin-price-data-fetcher/src/uploader.js | 38 +++++ 13 files changed, 732 insertions(+) create mode 100644 script/coin-price-data-fetcher/config/default.js create mode 100644 script/coin-price-data-fetcher/config/develop.js create mode 100644 script/coin-price-data-fetcher/package.json create mode 100644 script/coin-price-data-fetcher/src/api.js create mode 100644 script/coin-price-data-fetcher/src/fetcher.js create mode 100644 script/coin-price-data-fetcher/src/fetcher.test.js create mode 100644 script/coin-price-data-fetcher/src/fiat-exchange-rate.js create mode 100644 script/coin-price-data-fetcher/src/index.js create mode 100644 script/coin-price-data-fetcher/src/logger.js create mode 100644 script/coin-price-data-fetcher/src/monitor.js create mode 100644 script/coin-price-data-fetcher/src/sign.js create mode 100644 script/coin-price-data-fetcher/src/types.js create mode 100644 script/coin-price-data-fetcher/src/uploader.js diff --git a/script/coin-price-data-fetcher/config/default.js b/script/coin-price-data-fetcher/config/default.js new file mode 100644 index 00000000..29398184 --- /dev/null +++ b/script/coin-price-data-fetcher/config/default.js @@ -0,0 +1,35 @@ +module.exports = { + sourceCurrency: 'ADA', + targetFiatCurrencies: ['USD', 'JPY', 'EUR', 'CNY', 'KRW'], + targetCryptoCurrencies: ['ETH', 'BTC'], + logger: { + level: 'info' + }, + exchangeRateRefreshInterval: 10*60*1000, + serviceDataFreshnessThreshold: 2*60*1000, + // monitor allows 10% difference between the price from the service and directly from the API + monitorDiscrepancyThreshold: 0.1, + fetchTimeout: 30*1000, + apiKeys: { + cryptocompare: process.env.API_KEY_CRYPTOCOMPARE, + coinlayer: process.env.API_KEY_COINLAYER, + coinmarketcap: process.env.API_KEY_COINMARKETCAP, + coinapi: process.env.API_KEY_COINAPI, + nomics: process.env.API_KEY_NOMICS, + cryptoapis: process.env.API_KEY_CRYPTOAPIS, + openexchangerates: process.env.API_KEY_OPENEXCHANGERATES, + }, + + privKeyData: process.env.COIN_PRICE_PRIV_KEY, + pubKeyData: process.env.COIN_PRICE_PUB_KEY, + + s3: { + region: process.env.PRICE_DATA_S3_REGION, + bucketName: process.env.PRICE_DATA_S3_BUCKET_NAME, + accessKeyId: process.env.PRICE_DATA_S3_ACCESS_KEY_ID, + secretAccessKey: process.env.PRICE_DATA_S3_SECRET_ACCESS_KEY, + }, + + exchangeRateCachePath: '/tmp/exchange-rates.json', + exchangeRateCacheTime: 60*60*1000, // 1 hour +} diff --git a/script/coin-price-data-fetcher/config/develop.js b/script/coin-price-data-fetcher/config/develop.js new file mode 100644 index 00000000..0ced0e1c --- /dev/null +++ b/script/coin-price-data-fetcher/config/develop.js @@ -0,0 +1,26 @@ +module.exports = { + fetcherProviders: [ + //'cryptocompare', + //'coinlayer', + //'coinmarketcap', + //'coinapi', + //'coinpaprika', + //'nomics', + 'cryptonator', + //'shrimpy', + //'cryptoapis', + // 'badMockApi', + ], + monitorProviders: [ + // 'cryptocompare', + // 'coinlayer', + // 'coinmarketcap', + // 'coinapi', + // 'coinpaprika', + // 'nomics', + // 'cryptonator', + 'shrimpy', + // 'cryptoapis', + // 'badMockApi', + ], +}; diff --git a/script/coin-price-data-fetcher/package.json b/script/coin-price-data-fetcher/package.json new file mode 100644 index 00000000..09eef4f4 --- /dev/null +++ b/script/coin-price-data-fetcher/package.json @@ -0,0 +1,32 @@ +{ + "name": "coin-price-data-fetcher", + "version": "0.0.1", + "description": "coin price data fetcher", + "main": "index.js", + "scripts": { + "flow": "flow", + "postinstall": "npm run flow-remove-types", + "flow-remove-types": "flow-remove-types -d ./flow-files/config --all --pretty config/ && flow-remove-types -d ./flow-files/src --all --pretty src/", + "start-fetcher": "NODE_ENV=${NODE_ENV:-develop} node ./flow-files/src/index.js", + "start-monitor": "NODE_ENV=${NODE_ENV:-develop} node ./flow-files/src/index.js monitor", + "test": "npm run flow-remove-types && jest ./flow-files/src" + }, + "author": "", + "license": "ISC", + "dependencies": { + "@emurgo/cardano-serialization-lib-nodejs": "^9.1.4", + "aws-sdk": "^2.1057.0", + "bunyan": "^1.8.12", + "config": "^3.2.2", + "fetch-timeout": "0.0.2", + "flow": "^0.2.3", + "flow-remove-types": "^2.106.2", + "node-fetch": "^2.6.0" + }, + "devDependencies": { + "jest": "^24.9.0" + }, + "jest": { + "testEnvironment": "node" + } +} diff --git a/script/coin-price-data-fetcher/src/api.js b/script/coin-price-data-fetcher/src/api.js new file mode 100644 index 00000000..a795c508 --- /dev/null +++ b/script/coin-price-data-fetcher/src/api.js @@ -0,0 +1,142 @@ +// @flow +// This module defines the API providers. +import type { PairRate } from './types'; + +class ErrorResponse extends Error { + constructor(msg?: string) { + super(msg || 'Unexpected resposne'); + } +} + +export type FetchFunc = (url: string, headers?: { [header:string]: string }) => + Promise; +export type ApiFunc = (fetch: FetchFunc, apiKey: string) => Promise>; + +// Provides ADA-all +const cryptocompare: ApiFunc = async (fetch, apiKey) => { + const response = await fetch(`https://min-api.cryptocompare.com/data/price?fsym=ADA&tsyms=USD,JPY,EUR,CNY,KRW,BTC,ETH&api_key=${apiKey}`); + if (response.Response === 'Error') { + throw new ErrorResponse(); + } + return ['USD', 'JPY', 'EUR', 'CNY', 'KRW', 'BTC', 'ETH'].map(to => + ({ from: 'ADA', to, price: response[to] })); +}; + +// Provides ADA-USD, BTC-USD, ETH-USD +const coinlayer: ApiFunc = async (fetch, apiKey) => { + const response = await fetch(`http://api.coinlayer.com/api/live?access_key=${apiKey}&symbols=ADA,BTC,ETH&target=USD`); + if (response.success !== true) { + throw new ErrorResponse(); + } + return Object.keys(response.rates).map(from => ({ from, to: 'USD', price: response.rates[from] })); +}; + +// Provides ADA-USD, BTC-USD, ETH-USD +const coinmarketcap: ApiFunc = async (fetch, apiKey) => { + const response = await fetch(`https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest?symbol=ADA,ETH,BTC&convert=USD`, { + 'X-CMC_PRO_API_KEY': apiKey, + 'Accept': 'application/json' + }); + if (response.status.error_code !== 0) { + throw new ErrorResponse(); + } + return ['ETH', 'ADA', 'BTC'].map(from => + ({ from, to: 'USD', price: response.data[from].quote.USD.price })); +}; + +// Provides ADA-USD, ADA-BTC, ADA-ETH +const coinapi: ApiFunc = async (fetch, apiKey) => { + return Promise.all(['USD', 'BTC', 'ETH'].map(async (to) => { + const response = await fetch(`https://rest.coinapi.io/v1/exchangerate/ADA/${to}`, + { 'X-CoinAPI-Key': apiKey }); + return { from: 'ADA', to, price: response.rate }; + })); +}; + +// Provides ADA-all +const coinpaprika: ApiFunc = async (fetch, _apiKey) => { + const response = await fetch('https://api.coinpaprika.com/v1/tickers/ada-cardano?quotes=USD,BTC,ETH,JPY,EUR,CNY,KRW'); + return ['USD', 'JPY', 'EUR', 'CNY', 'KRW', 'BTC', 'ETH'].map(to => + ({ from: 'ADA', to, price: response.quotes[to].price })); +}; + +// Provides ADA-USD,BTC,ETH +const nomics: ApiFunc = async (fetch, apiKey) => { + let result; + // fetch ETH,BTC to ADA rate and take the reciprocals + let response = await fetch(`https://api.nomics.com/v1/currencies/ticker?key=${apiKey}&ids=ETH,BTC&convert=ADA&interval=1h`); + result = response.map(data => + ({ from: 'ADA', to: data.symbol, price: 1/Number(data.price) }) + ); + + // fetch ADA-USD + response = await fetch(`https://api.nomics.com/v1/currencies/ticker?key=${apiKey}&ids=ADA&convert=USD&interval=1h`); + result.push({ from: 'ADA', to: 'USD', price: Number(response[0].price) }); + + return result; +}; + +// Provides ADA-all except CNY +const cryptonator: ApiFunc = async (fetch, apiKey) => { + // cryptonator doesn't have CNY or KRW + return Promise.all(['usd', 'jpy', 'eur', 'btc', 'eth'].map(async (to) => { + const response = await fetch(`https://api.cryptonator.com/api/ticker/ada-${to}`); + if (response.success !== true) { + throw new ErrorResponse(); + } + return { from: 'ADA', to: response.ticker.target, price: Number(response.ticker.price) }; + })); +}; + +// Provides ADA-USD,BTC,ETH +const shrimpy: ApiFunc = async (fetch, _apiKey) => { + const response = await fetch('https://dev-api.shrimpy.io/v1/exchanges/kraken/ticker'); + const adaData = response.find(data => data.symbol === 'ADA'); + const ethData = response.find(data => data.symbol === 'ETH'); + + return [ + { from: 'ADA', to: 'USD', price: Number(adaData.priceUsd) }, + { from: 'ADA', to: 'BTC', price: Number(adaData.priceBtc) }, + { from: 'ADA', to: 'ETH', price: Number(adaData.priceUsd) / Number(ethData.priceUsd) } + ]; +}; + +// Provides ADA-BTC,ETH,USD,EUR +const cryptoapis: ApiFunc = async (fetch, apiKey) => { + /* source: + curl -H 'X-API-Key:b67bf0950e65a556579e2d27e1c3914d44158628' 'https://api.cryptoapis.io/v1/assets/meta?skip=skip&limit=limit' | python -mjson.tool */ + const ids = { + ADA: '5b1ea92e584bf5002013062d', + BTC: '5b1ea92e584bf50020130612', + USD: '5b1ea92e584bf50020130615', + EUR: '5b1ea92e584bf5002013061a', + ETH: '5b755dacd5dd99000b3d92b2', + } + return Promise.all(['BTC', 'ETH', 'USD'].map(async (to) => { + const response = await fetch(`https://api.cryptoapis.io/v1/exchange-rates/${ids.ADA}/${ids[to]}`, + { 'X-API-Key': apiKey }); + return { from: 'ADA', to, price: response.payload.weightedAveragePrice } + })); +}; + +// A mock API that always fails, for testing +const badMockApi: ApiFunc = async (_fetch, _apiKey) => { + throw new Error('bad mock API fails'); +}; + +module.exports = { + ErrorResponse, + providers: { + cryptocompare, + coinlayer, + coinmarketcap, + coinapi, + coinpaprika, + nomics, + cryptonator, + shrimpy, + cryptoapis, + badMockApi, + }, +}; + diff --git a/script/coin-price-data-fetcher/src/fetcher.js b/script/coin-price-data-fetcher/src/fetcher.js new file mode 100644 index 00000000..da036dbd --- /dev/null +++ b/script/coin-price-data-fetcher/src/fetcher.js @@ -0,0 +1,115 @@ +// @flow +// This modules fetches data using APIs defined in api.js. +const config = require('config'); +const fetch = require('fetch-timeout'); +const api = require('./api'); +const logger = require('./logger'); +const exchangeRate = require('./fiat-exchange-rate'); + +import type { PairRate } from './types'; + +async function queryApi(apiName: string): Promise> { + let retry = 0; + + async function _fetch(url, headers) { + let __logger = logger.child({ apiName, url, retry }); + __logger.info('fetching ' + url, headers); + const response = await fetch(url, { headers }, config.fetchTimeout, 'fetch timeout'); + if (!response.ok) { + __logger.error('error', response.status, response.statusText); + throw new Error('Fetch error'); + } + const json = await response.json(); + __logger.info('response', json); + return json; + } + + for (;;) { + const _logger = logger.child({ api: apiName, retry }); + + try { + const result = await api.providers[apiName](_fetch, config.apiKeys[apiName]); + _logger.info(`got data from ${apiName}`, { result }); + return result; + } catch (error) { + _logger.error(error); + if (retry === 2) { + throw new Error(`failed to get data from ${apiName}`) + } + retry++; + } + } //for + throw new Error('will never reach here'); +} + +/* + Each API provider has different capabilities. Some lacks exchange rates + against certain fiat currencies. This function calculates missing exchanges so + that the returned ticker set include pairs of ADA and all target currencies. +*/ +function normalizeQueryResult(queryResult: Array): Array { + function findPair(from, to) { + return queryResult.find(pair => (pair.from === from) && (pair.to === to)); + } + const result = []; + const adaUsd = findPair('ADA', 'USD'); + if (!adaUsd) { + throw new Error('missing ADA-USD rate'); + } + result.push(adaUsd); + + for (const fiat of config.targetFiatCurrencies.filter(s => s !== 'USD')) { + const pair = findPair('ADA', fiat); + if (pair) { + result.push(pair); + } else { + const price = adaUsd.price * exchangeRate.getFromUsd(fiat); + result.push({ from: 'ADA', to: fiat, price }); + } + } + + for (const crypto of config.targetCryptoCurrencies) { + const pair = findPair('ADA', crypto); + if (pair) { + result.push(pair); + continue; + } + + const cryptoUsd = findPair(crypto, 'USD'); + if (!cryptoUsd) { + throw new Error(`missing ${crypto} rate`); + } + const price = adaUsd.price/cryptoUsd.price; + result.push({ from: 'ADA', to: crypto, price }); + } + + return result; +} + +async function queryAndNormalize(apiName: string): Promise> { + let apiResult; + try { + apiResult = await queryApi(apiName); + } catch (_) { + return null; + } + return normalizeQueryResult(apiResult); +} + +module.exports = { queryAndNormalize }; + +//NODE_ENV=development node fetcher.js | ./node_modules/.bin/bunyan -c 'this.msg.match(/got data from/)' +if (require.main === module) { + (async () => { + await exchangeRate.start(); + for (const apiName in api.providers) { + console.log('-'.repeat(30)+apiName+'-'.repeat(30)); + const result = await queryApi(apiName); + logger.info('normalized result', normalizeQueryResult(result)); + } + process.exit(0); + })().catch(error => { + console.error(error); + }); +} + diff --git a/script/coin-price-data-fetcher/src/fetcher.test.js b/script/coin-price-data-fetcher/src/fetcher.test.js new file mode 100644 index 00000000..9265a86b --- /dev/null +++ b/script/coin-price-data-fetcher/src/fetcher.test.js @@ -0,0 +1,49 @@ +// @flow +const config = require('config'); +const api = require('./api'); +const exchangeRate = require('./fiat-exchange-rate'); +const fetcher = require('./fetcher'); + +import type { PairRate } from './types'; + +const ALLOWED_DEVIATION_FROM_MEDIAN = 0.2; //20% + +beforeAll(async () => { + await exchangeRate.start(); +}); + +afterAll(() => { + exchangeRate.stop(); +}); + +// We need this because flow doesn't recognize the timeout parameter: +declare function test(name: string, fn: void => Promise, timeout?: number): void; + +jest.setTimeout(60*1000); + +test('consistency of data fetched from different APIs', async () => { + const providers = Object.keys(api.providers); + const rawMultiSourceData: Array> = await Promise.all( + providers.map(apiName => fetcher.queryAndNormalize(apiName))); + const multiSourceData: Array> = (rawMultiSourceData.filter(d => d !== null): any); + for (const to of [...config.targetFiatCurrencies, ...config.targetCryptoCurrencies]) { + const prices = multiSourceData.map(data => { + const pair = data.find(pair => pair.to === to); + if (!pair) { + throw new Error('missing data'); + } + return pair.price; + }); + + prices.sort((a, b) => a - b); + + const median = prices[Math.floor(prices.length / 2)]; + + for (const price of prices) { + if (Math.abs(price - median) > ALLOWED_DEVIATION_FROM_MEDIAN * median) { + throw new Error(`Prices of ${to} is inconsistent: ${price} outlies from [${prices.join(',')}].\n`+ + `Data from ${providers.join(' ')}:\n${JSON.stringify(multiSourceData)}`); + } + } + } +}); diff --git a/script/coin-price-data-fetcher/src/fiat-exchange-rate.js b/script/coin-price-data-fetcher/src/fiat-exchange-rate.js new file mode 100644 index 00000000..0b8a63d8 --- /dev/null +++ b/script/coin-price-data-fetcher/src/fiat-exchange-rate.js @@ -0,0 +1,108 @@ +// @flow +// This module provides fiat exchange rates. +const fs = require('fs'); +const config = require('config'); +const fetch = require('fetch-timeout'); +const logger = require('./logger'); + +type Rates = { [to: string]: number }; +type Response = { timestamp: number, rates: Rates }; +type Cache = {| response: Response, timestamp: number, version: number |}; + +let rates: ?Rates; +let timeoutId: TimeoutID; + +function stop() { + clearTimeout(timeoutId); +} + +async function start(): Promise { + for (let retry = 0; ; retry++) { + const _logger = logger.child({ exchangeRateRequest: retry }); + + try { + _logger.info('requesting exchange rates'); + + // Try to load from cache to avoid exceeding limits of the API. The API + // data is hourly anyway. + try { + const cache: Cache = require(config.exchangeRateCachePath); + _logger.info('cached exchange rate', cache); + /* + Sometimes openexchangerates keeps returning a stale timestamp for a while. + When this happens, we don't want to re-query every time to avoid + exhausting the query quota. + */ + if (Date.now() - cache.response.timestamp * 1000 < config.exchangeRateCacheTime || + Date.now() - cache.timestamp < config.exchangeRateRefreshInterval + ) { + _logger.info('use cached exchange rates'); + rates = cache.response.rates; + break; + } + _logger.info('cached exchange rates are stale'); + } catch (_) { + _logger.info('no exchange rate cache'); + } + + + const response = await fetch( + `https://openexchangerates.org/api/latest.json?app_id=${config.apiKeys.openexchangerates}&base=USD&symbols=EUR,CNY,JPY,KRW`, + {}, + config.fetchTimeout, + 'fetch timeout' + ); + if (!response.ok) { + throw new Error(`error response: ${response.status} ${response.statusText}`); + } + const json = await response.json(); + _logger.info('response', json); + if (!validateRates(json.rates)) { + throw new Error('invalid response'); + } + rates = json.rates; + let cache: Cache = { response: json, timestamp: Date.now(), version: 1 }; + try { + fs.writeFileSync(config.exchangeRateCachePath, JSON.stringify(cache)); + } catch (error) { + _logger.error('failed to write cache', error); + } + break; + } catch (error) { + _logger.error(error); + if (retry === 2) { + throw new Error('failed to query exchange rate'); + } + await new Promise(resolve => { setTimeout(resolve, 10*1000); }); + } + } + timeoutId = setTimeout(start, config.exchangeRateRefreshInterval); +} + +function getFromUsd(to: string): number { + if (!rates) { + throw new Error('fiat exchange rates are unavailable'); + } + return rates[to]; +} + +function validateRates(rates/*: Object*/) { + for (const currency of config.targetFiatCurrencies.filter(c => c !== 'USD')) { + if (typeof rates[currency] !== 'number') { + return false; + } + } + return true; +} + +module.exports = { start, stop, getFromUsd }; + +//NODE_ENV=develop node src/fiat-exchange-rate.js | ./node_modules/.bin/bunyan +if (require.main === module) { + (async () => { + await start(); + console.log('started'); + })().catch(error => { + console.error(error); + }); +} diff --git a/script/coin-price-data-fetcher/src/index.js b/script/coin-price-data-fetcher/src/index.js new file mode 100644 index 00000000..f0e5f7d8 --- /dev/null +++ b/script/coin-price-data-fetcher/src/index.js @@ -0,0 +1,61 @@ +// @flow +const config = require('config'); +const { PrivateKey } = require('@emurgo/cardano-serialization-lib-nodejs'); +const exchangeRate = require('./fiat-exchange-rate'); +const fetcher = require('./fetcher'); +const logger = require('./logger'); +const sign = require('./sign'); +const uploader = require('./uploader'); +const monitor = require('./monitor'); + +import type { Ticker } from './types'; + +function median(numbers: Array): number { + numbers.sort((a, b) => a - b); + return numbers[Math.floor(numbers.length/2)]; +} + +async function main() { + let mode = (process.argv[2] === 'monitor') ? 'monitor' : 'fetcher'; + + logger.info(`start ${mode}`); + + await exchangeRate.start(); + + const providers = (mode === 'monitor') ? config.monitorProviders : config.fetcherProviders; + const multiSourceData = (await Promise.all(providers.map(apiName => + fetcher.queryAndNormalize(apiName)))).filter(data => data !== null); + logger.info('fetched data', multiSourceData); + + const ticker: Ticker = { + from: 'ADA', + timestamp: Math.floor(Date.now()/1000) * 1000, + prices: {} + }; + + // take the median over data from mulitiple APIs + for (const to of [...config.targetFiatCurrencies, ...config.targetCryptoCurrencies]) { + const price = median(multiSourceData.map(data => + data.find(pair => pair.to === to).price + )); + ticker.prices[to] = price; + } + ticker.signature = sign.sign( + ticker, sign.serializeTicker, + PrivateKey.from_extended_bytes(Buffer.from(config.privKeyData, 'hex')) + ); + logger.info('uploading ticker', ticker); + + if (mode === 'monitor') { + await monitor.monitor(ticker); + } else { + await uploader.upload( ticker ); + } + process.exit(0); +} + +main().catch(error => { + logger.error(error); + process.exit(1); +}); + diff --git a/script/coin-price-data-fetcher/src/logger.js b/script/coin-price-data-fetcher/src/logger.js new file mode 100644 index 00000000..27a735fb --- /dev/null +++ b/script/coin-price-data-fetcher/src/logger.js @@ -0,0 +1,6 @@ +//@flow +const bunyan = require('bunyan'); +const config = require('config'); +const logger = bunyan.createLogger({name: 'price data fetcher', level: config.logger.level}); + +module.exports = logger; diff --git a/script/coin-price-data-fetcher/src/monitor.js b/script/coin-price-data-fetcher/src/monitor.js new file mode 100644 index 00000000..94586465 --- /dev/null +++ b/script/coin-price-data-fetcher/src/monitor.js @@ -0,0 +1,58 @@ +// @flow +const config = require('config'); +const fetch = require('fetch-timeout'); +const { PublicKey } = require('cardano-wallet'); +const logger = require('./logger'); +const sign = require('./sign'); + +import type { Ticker } from './types'; + +async function monitor(tickerFromApi: Ticker): Promise { + let tickerFromService: ?Ticker; + + for (let retry = 0; retry < 3; retry++) { + try { + logger.info('fetch price data from the service', { retry }); + const response = await fetch( + `${config.serviceEndPointUrlPrefix}/price/ADA/current`, + {}, + config.fetchTimeout, + 'fetch timeout' + ); + if (!response.ok) { + throw new Error(`error response ${response.status}`); + } + tickerFromService = (await response.json()).ticker; + logger.info('price data from the service', tickerFromService); + break; + } catch (error) { + logger.error(error); + } + } + + if (!tickerFromService) { + throw new Error('service is unavailable'); + } + + // Check signature. + if (!sign.verify(tickerFromService, sign.serializeTicker, tickerFromService.signature, + PublicKey.from_hex(config.pubKeyData))) { + throw new Error('ticker signature error'); + } + + // Check refreshness. + if (Date.now() - tickerFromService.timestamp > config.serviceDataFreshnessThreshold) { + throw new Error('data are stale'); + } + + // Check price values against data directly fetched from API + for (const currency of [...config.targetFiatCurrencies, ...config.targetCryptoCurrencies]) { + const priceFromService = tickerFromService.prices[currency]; + const priceFromApi = tickerFromApi.prices[currency]; + if (Math.abs(priceFromService - priceFromApi) > config.monitorDiscrepancyThreshold * priceFromApi) { + throw new Error(`prices of ${currency} differs too much`); + } + } +} + +module.exports = { monitor }; diff --git a/script/coin-price-data-fetcher/src/sign.js b/script/coin-price-data-fetcher/src/sign.js new file mode 100644 index 00000000..a8cc1d35 --- /dev/null +++ b/script/coin-price-data-fetcher/src/sign.js @@ -0,0 +1,49 @@ +// @flow +const { PrivateKey, PublicKey } = require("@emurgo/cardano-serialization-lib-nodejs"); +/*:: import type { Ticker } from './types'; */ + +function serializeTicker(ticker/*: Ticker*/)/*: Buffer*/ { + return Buffer.from(ticker.from + + ticker.timestamp + + Object.keys(ticker.prices).sort().map(to => to + ticker.prices[to]).join(''), + 'utf8' + ); +} + +function sign( + obj/*: any*/, + serializer/*: any => Buffer*/, + privateKey/*: PrivateKey*/ +)/*: string*/ { + return privateKey.sign(serializer(obj)).to_hex(); +} + +function verify( + obj/*: any*/, + serializer/*: any => Buffer*/, + signatureHex/*: string*/, + publicKey/*: PublicKey*/ +)/*: boolean*/ { + return publicKey.verify( + serializer(obj), + CardanoWasm.Ed25519Signature.from_bytes(Buffer.from(signatureHex, "hex")) + ); +} + +module.exports = { serializeTicker, sign, verify }; + +if (require.main === module) { + const config = require('config'); + + if (process.argv[2] === 'verify') { + const ticker = JSON.parse(process.argv[3]); + const pubKey = PublicKey.from_hex(config.pubKeyData); + console.log(verify(ticker, serializeTicker, ticker.signature, pubKey)); + } else if (process.argv[2] === 'sign') { + const ticker = JSON.parse(process.argv[3]); + const privKey = PrivateKey.from_hex(config.privKeyData); + console.log(sign(ticker, serializeTicker, privKey)); + } else { + console.log('node sign.js verify|sign '); + } +} diff --git a/script/coin-price-data-fetcher/src/types.js b/script/coin-price-data-fetcher/src/types.js new file mode 100644 index 00000000..c9617d8d --- /dev/null +++ b/script/coin-price-data-fetcher/src/types.js @@ -0,0 +1,13 @@ +// @flow +export type Ticker = {| + from: string, + timestamp: number, + signature?: string, + prices: { [targetCurrency:string]: number } +|}; + +export type PairRate = {| + from: string, + to: string, + price: number, +|}; diff --git a/script/coin-price-data-fetcher/src/uploader.js b/script/coin-price-data-fetcher/src/uploader.js new file mode 100644 index 00000000..02a1f975 --- /dev/null +++ b/script/coin-price-data-fetcher/src/uploader.js @@ -0,0 +1,38 @@ +// @flow +const util = require('util'); +const config = require('config'); +const AWS = require('aws-sdk'); +const logger = require('./logger'); + +import type { Ticker } from './types'; + +AWS.config.update({region: config.get("s3.region")}); + +const S3 = new AWS.S3({ + accessKeyId: config.get('s3.accessKeyId'), + secretAccessKey: config.get('s3.secretAccessKey'), +}); + +const RETRY_COUNT = 3; + +async function upload(ticker: Ticker): Promise { + const fileName = `prices-${ticker.from}-${ticker.timestamp}.json`; + const uploadParams = { + Body: JSON.stringify(ticker), + Key: fileName, + Bucket: config.get('s3.bucketName'), + }; + for (let i = 0; i < RETRY_COUNT; i++) { + let resp; + try { + resp = await util.promisify(S3.upload.bind(S3))(uploadParams); + } catch (error) { + logger.error('upload failed:', error); + continue; + } + logger.info('price data uploaded:', resp); + break; + } +} + +module.exports = { upload };