From ad2436ec9b714e028ba7b19d6dd1c7e6dde1b5eb Mon Sep 17 00:00:00 2001 From: Vladan Paunovic Date: Tue, 25 Apr 2023 18:13:26 +0200 Subject: [PATCH] chore: optimise db (#124) --- .github/workflows/sync_all_coins.yml | 2 +- .../workflows/sync_all_coins_description.yml | 2 +- .github/workflows/sync_available_tokens.yml | 2 +- components/helpers/GoogleAnalytics.js | 12 +- scripts/sync_all_coins.js | 113 ++++++++++++------ scripts/sync_all_coins_description.js | 45 +++---- scripts/sync_available_coins.js | 19 ++- 7 files changed, 127 insertions(+), 68 deletions(-) diff --git a/.github/workflows/sync_all_coins.yml b/.github/workflows/sync_all_coins.yml index c3f2976..46618de 100644 --- a/.github/workflows/sync_all_coins.yml +++ b/.github/workflows/sync_all_coins.yml @@ -2,7 +2,7 @@ name: Sync all coins once a day on: schedule: - - cron: "0 4 */7 * *" + - cron: "0 4 */1 * *" # Run at 04:00 UTC every day workflow_dispatch: jobs: diff --git a/.github/workflows/sync_all_coins_description.yml b/.github/workflows/sync_all_coins_description.yml index 9c444b3..da3c53e 100644 --- a/.github/workflows/sync_all_coins_description.yml +++ b/.github/workflows/sync_all_coins_description.yml @@ -2,7 +2,7 @@ name: Sync all coins metadata once a month on: schedule: - - cron: "0 7 1 * *" + - cron: "0 7 1 * *" # Every 1st day of the month at 7am UTC workflow_dispatch: jobs: diff --git a/.github/workflows/sync_available_tokens.yml b/.github/workflows/sync_available_tokens.yml index faa1cd3..1801d5f 100644 --- a/.github/workflows/sync_available_tokens.yml +++ b/.github/workflows/sync_available_tokens.yml @@ -2,7 +2,7 @@ name: Sync all available tokens on: schedule: - - cron: "0 1 */7 * *" + - cron: "0 1 */1 * *" # Run at 01:00 UTC every day workflow_dispatch: jobs: diff --git a/components/helpers/GoogleAnalytics.js b/components/helpers/GoogleAnalytics.js index c918a45..263d32f 100644 --- a/components/helpers/GoogleAnalytics.js +++ b/components/helpers/GoogleAnalytics.js @@ -2,12 +2,16 @@ import { GA_TRACKING_ID } from "../../config"; // log the pageview with their URL export const pageview = (url) => { - window.gtag("config", GA_TRACKING_ID, { - page_path: url, - }); + if (window?.gtag) { + window.gtag("config", GA_TRACKING_ID, { + page_path: url, + }); + } }; // log specific events happening. export const event = ({ action, params }) => { - window.gtag("event", action, params); + if (window?.gtag) { + window?.gtag("event", action, params); + } }; diff --git a/scripts/sync_all_coins.js b/scripts/sync_all_coins.js index a913cfc..b514b2e 100644 --- a/scripts/sync_all_coins.js +++ b/scripts/sync_all_coins.js @@ -6,7 +6,11 @@ initSentry(); /** @type {import('@prisma/client').PrismaClient} */ const prismaClient = global.prisma || new PrismaClient(); -const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const sleep = (ms) => + new Promise((resolve) => { + console.log(`Sleeping for ${ms / 1000}s...`); + setTimeout(resolve, ms); + }); const CRYPTOCOMPARE_API_KEY = process.env.CRYPTOCOMPARE_API_KEY; const COINCAP_API_KEY = process.env.COINCAP_API_KEY; @@ -55,11 +59,6 @@ const getCoinPrice = async (coinSymbol) => { if (coinDataResponse.status === 429) { const SLEEP_TIMEOUT_70_SEC = 79000; - console.log( - `CoinGecko API Rate limit exceeded, sleeping for ${ - SLEEP_TIMEOUT_70_SEC / 1000 - }s and retrying` - ); await sleep(SLEEP_TIMEOUT_70_SEC); return getCoinPrice(coinSymbol); } @@ -76,39 +75,47 @@ const getCoinPrice = async (coinSymbol) => { async function main() { const availableCoins = await getAllAvailableTokens(); + const existingCoins = await prismaClient.cryptocurrency.findMany({ + select: { coinId: true }, + }); + const existingCoinIds = existingCoins.map((coin) => coin.coinId); - // delete all coins from the database that are not in the availableCoins list - await prismaClient.cryptocurrency.deleteMany({ - where: { - coinId: { - notIn: availableCoins.map((coin) => coin.id), + const coinsToDelete = existingCoinIds.filter( + (coinId) => !availableCoins.some((coin) => coin.id === coinId) + ); + + // Delete coins not in availableCoins list + if (coinsToDelete.length > 0) { + console.log(`Deleting ${coinsToDelete.length} coins...`); + await prismaClient.cryptocurrency.deleteMany({ + where: { + coinId: { + in: coinsToDelete, + }, }, - }, - }); + }); + } - // for each coin in availableCoins fetch the coin data from the external APIs - // and store it in the database - for (let index = 0; index < availableCoins.length; index++) { + const coinsToCreate = []; + const coinsToUpdate = []; + + // Check if a coin is new or already exists in the database + for (const coin of availableCoins) { // sleep every 10 requests to avoid rate limits - if (index % 100 === 0) { - const SLEEP_TIMEOUT_10_SEC = 10000; - console.log( - `Sleeping for ${SLEEP_TIMEOUT_10_SEC / 1000}s to avoid rate limits` - ); - await sleep(SLEEP_TIMEOUT_10_SEC); - } + const coinIndex = availableCoins.indexOf(coin); + const coinPrices = await getCoinPrice(coin.symbol); - const coin = availableCoins[index]; + if (coinIndex % 200 === 0 && coinIndex !== 0) { + await sleep(10000); // sleep for 10 seconds + } - const coinPrices = await getCoinPrice(coin.symbol); + console.log( + `Processing #${coinIndex + 1}/${availableCoins.length} ${coin.id}...` + ); // if fields in coinData are missing, skip it if (!coin.name || !coin.symbol || !coin.id) { - console.log( - `Skipping #${index + 1}/${availableCoins.length} ${ - coin.id - } due to missing fields` - ); + console.log(`Skipping ${coin.id} due to missing fields`); continue; } @@ -119,21 +126,53 @@ async function main() { currentPrice: parseFloat(coin.priceUsd || 0), marketCapRank: parseInt(coin.rank), image: `https://assets.coincap.io/assets/icons/${coin.symbol.toLowerCase()}@2x.png`, + prices: coinPrices || [], }; - payload.prices = coinPrices || []; + if (existingCoinIds.includes(coin.id)) { + coinsToUpdate.push(payload); + } else { + coinsToCreate.push(payload); + } + } - // store cryptocurrencies in the database - console.log(`Storing #${index + 1}/${availableCoins.length} ${coin.id}`); + // Create new coins + if (coinsToCreate.length > 0) { + console.log(`Creating ${coinsToCreate.length} coins...`); + await prismaClient.cryptocurrency.createMany({ + data: coinsToCreate, + }); + } - await prismaClient.cryptocurrency.upsert({ + // Update existing coins + const updatePromises = coinsToUpdate.map((coin) => { + return prismaClient.cryptocurrency.update({ where: { - coinId: coin.id, + coinId: coin.coinId, + }, + data: { + currentPrice: parseFloat(coin.priceUsd || 0), + prices: coin.prices, }, - update: payload, - create: payload, }); + }); + + if (updatePromises.length > 0) { + console.log(`Updating ${coinsToUpdate.length} coins...`); + // show progress on update + for (const promise of updatePromises) { + const promiseIndex = updatePromises.indexOf(promise); + // sleep every 10 requests to avoid database failures + if (promiseIndex % 200 === 0 && promiseIndex !== 0) { + await sleep(10000); // sleep for 10 seconds + } + + console.log(`Updating coin ${promiseIndex + 1}/${updatePromises.length}`); + await promise; + } } + + console.log("Done!"); } main(); diff --git a/scripts/sync_all_coins_description.js b/scripts/sync_all_coins_description.js index ad6d55c..968dfbf 100644 --- a/scripts/sync_all_coins_description.js +++ b/scripts/sync_all_coins_description.js @@ -53,10 +53,28 @@ const getAllAvailableTokens = async () => { async function main() { const availableCoins = await getAllAvailableTokens(); - let discovered = 0; - for (let index = 0; index < availableCoins.length; index++) { - const coin = availableCoins[index]; + const symbols = availableCoins.map((coin) => coin.SYMBOL); + const coinsFromDb = await prismaClient.cryptocurrency.findMany({ + where: { + symbol: { + in: symbols, + }, + }, + }); + + const coinsToUpdate = availableCoins.filter((apiCoin) => { + const dbCoin = coinsFromDb.find((coin) => coin.symbol === apiCoin.SYMBOL); + return ( + dbCoin && + (dbCoin.description !== apiCoin.ASSET_DESCRIPTION || + dbCoin.descriptionSummary !== apiCoin.ASSET_DESCRIPTION_SUMMARY) + ); + }); + console.log(`Updating: ${coinsToUpdate.length} coins`); + let discovered = 0; + for (let index = 0; index < coinsToUpdate.length; index++) { + const coin = coinsToUpdate[index]; const { ASSET_DESCRIPTION, ASSET_DESCRIPTION_SUMMARY, @@ -65,24 +83,9 @@ async function main() { SYMBOL, } = coin; - const findCoin = await prismaClient.cryptocurrency.findFirst({ - where: { - symbol: SYMBOL, - }, - }); - - if (!findCoin) { - console.log( - `#${index + 1}/${ - availableCoins.length - } - Not found in the database, skipping...` - ); - continue; - } - console.log( `#${index + 1}/${ - availableCoins.length + coinsToUpdate.length } - Updating coin ${NAME} (${SYMBOL})` ); @@ -98,10 +101,10 @@ async function main() { discovered += 1; } - // calculate the percentage of found tokens + // calculate the percentage of updated tokens const percentage = (discovered / availableCoins.length) * 100; console.log( - `Discovered ${discovered}/${ + `Updated ${discovered}/${ availableCoins.length } tokens (${percentage.toFixed(2)}%)` ); diff --git a/scripts/sync_available_coins.js b/scripts/sync_available_coins.js index 7b211d7..dec9ce3 100644 --- a/scripts/sync_available_coins.js +++ b/scripts/sync_available_coins.js @@ -3,10 +3,16 @@ const { PrismaClient } = require("@prisma/client"); const initSentry = require("./initSentry"); initSentry(); +/** + * This script is used to sync the available coins from CoinGecko to Redis. + * This is used to populate the available coins in the frontend. + * This script is run on a cron job. + */ + /** @type {import('@prisma/client').PrismaClient} */ const prismaClient = global.prisma || new PrismaClient(); -const mapCoinGeckoResponseToRedis = (coins) => { +const mapCoinGeckoResponseToKeyValue = (coins) => { const redisCoins = coins.map((coin) => ({ id: coin.coinId, coinId: coin.coinId, @@ -20,10 +26,17 @@ const mapCoinGeckoResponseToRedis = (coins) => { async function main() { console.log("Fetching all available coins..."); - const allCoins = await prismaClient.cryptocurrency.findMany(); + const allCoins = await prismaClient.cryptocurrency.findMany({ + select: { + coinId: true, + symbol: true, + name: true, + marketCapRank: true, + }, + }); console.log("Mapping coins to redis format..."); - const mappedCoins = mapCoinGeckoResponseToRedis(allCoins); + const mappedCoins = mapCoinGeckoResponseToKeyValue(allCoins); console.log("Upserting coins to Prisma..."); const key = "availableTokens";