chore: optimise db (#124)

.github/workflows/sync_all_coins.yml

@@ -2,7 +2,7 @@ name: Sync all coins once a day
 
 on:
   schedule:
-    - cron: "0 4 */7 * *"
+    - cron: "0 4 */1 * *" # Run at 04:00 UTC every day
   workflow_dispatch:
 
 jobs:

.github/workflows/sync_all_coins_description.yml

@@ -2,7 +2,7 @@ name: Sync all coins metadata once a month
 
 on:
   schedule:
-    - cron: "0 7 1 * *"
+    - cron: "0 7 1 * *" # Every 1st day of the month at 7am UTC
   workflow_dispatch:
 
 jobs:

.github/workflows/sync_available_tokens.yml

@@ -2,7 +2,7 @@ name: Sync all available tokens
 
 on:
   schedule:
-    - cron: "0 1 */7 * *"
+    - cron: "0 1 */1 * *" # Run at 01:00 UTC every day
   workflow_dispatch:
 
 jobs:

components/helpers/GoogleAnalytics.js

@@ -2,12 +2,16 @@ import { GA_TRACKING_ID } from "../../config";
 
 // log the pageview with their URL
 export const pageview = (url) => {
-  window.gtag("config", GA_TRACKING_ID, {
-    page_path: url,
-  });
+  if (window?.gtag) {
+    window.gtag("config", GA_TRACKING_ID, {
+      page_path: url,
+    });
+  }
 };
 
 // log specific events happening.
 export const event = ({ action, params }) => {
-  window.gtag("event", action, params);
+  if (window?.gtag) {
+    window?.gtag("event", action, params);
+  }
 };

scripts/sync_all_coins.js

@@ -6,7 +6,11 @@ initSentry();
 /** @type {import('@prisma/client').PrismaClient} */
 const prismaClient = global.prisma || new PrismaClient();
 
-const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const sleep = (ms) =>
+  new Promise((resolve) => {
+    console.log(`Sleeping for ${ms / 1000}s...`);
+    setTimeout(resolve, ms);
+  });
 
 const CRYPTOCOMPARE_API_KEY = process.env.CRYPTOCOMPARE_API_KEY;
 const COINCAP_API_KEY = process.env.COINCAP_API_KEY;
@@ -55,11 +59,6 @@ const getCoinPrice = async (coinSymbol) => {
 
   if (coinDataResponse.status === 429) {
     const SLEEP_TIMEOUT_70_SEC = 79000;
-    console.log(
-      `CoinGecko API Rate limit exceeded, sleeping for ${
-        SLEEP_TIMEOUT_70_SEC / 1000
-      }s and retrying`
-    );
     await sleep(SLEEP_TIMEOUT_70_SEC);
     return getCoinPrice(coinSymbol);
   }
@@ -76,39 +75,47 @@ const getCoinPrice = async (coinSymbol) => {
 
 async function main() {
   const availableCoins = await getAllAvailableTokens();
+  const existingCoins = await prismaClient.cryptocurrency.findMany({
+    select: { coinId: true },
+  });
+  const existingCoinIds = existingCoins.map((coin) => coin.coinId);
 
-  // delete all coins from the database that are not in the availableCoins list
-  await prismaClient.cryptocurrency.deleteMany({
-    where: {
-      coinId: {
-        notIn: availableCoins.map((coin) => coin.id),
+  const coinsToDelete = existingCoinIds.filter(
+    (coinId) => !availableCoins.some((coin) => coin.id === coinId)
+  );
+
+  // Delete coins not in availableCoins list
+  if (coinsToDelete.length > 0) {
+    console.log(`Deleting ${coinsToDelete.length} coins...`);
+    await prismaClient.cryptocurrency.deleteMany({
+      where: {
+        coinId: {
+          in: coinsToDelete,
+        },
       },
-    },
-  });
+    });
+  }
 
-  // for each coin in availableCoins fetch the coin data from the external APIs
-  // and store it in the database
-  for (let index = 0; index < availableCoins.length; index++) {
+  const coinsToCreate = [];
+  const coinsToUpdate = [];
+
+  // Check if a coin is new or already exists in the database
+  for (const coin of availableCoins) {
     // sleep every 10 requests to avoid rate limits
-    if (index % 100 === 0) {
-      const SLEEP_TIMEOUT_10_SEC = 10000;
-      console.log(
-        `Sleeping for ${SLEEP_TIMEOUT_10_SEC / 1000}s to avoid rate limits`
-      );
-      await sleep(SLEEP_TIMEOUT_10_SEC);
-    }
+    const coinIndex = availableCoins.indexOf(coin);
+    const coinPrices = await getCoinPrice(coin.symbol);
 
-    const coin = availableCoins[index];
+    if (coinIndex % 200 === 0 && coinIndex !== 0) {
+      await sleep(10000); // sleep for 10 seconds
+    }
 
-    const coinPrices = await getCoinPrice(coin.symbol);
+    console.log(
+      `Processing #${coinIndex + 1}/${availableCoins.length} ${coin.id}...`
+    );
 
     // if fields in coinData are missing, skip it
     if (!coin.name || !coin.symbol || !coin.id) {
-      console.log(
-        `Skipping #${index + 1}/${availableCoins.length} ${
-          coin.id
-        } due to missing fields`
-      );
+      console.log(`Skipping ${coin.id} due to missing fields`);
       continue;
     }
 
@@ -119,21 +126,53 @@ async function main() {
       currentPrice: parseFloat(coin.priceUsd || 0),
       marketCapRank: parseInt(coin.rank),
       image: `https://assets.coincap.io/assets/icons/${coin.symbol.toLowerCase()}@2x.png`,
+      prices: coinPrices || [],
     };
 
-    payload.prices = coinPrices || [];
+    if (existingCoinIds.includes(coin.id)) {
+      coinsToUpdate.push(payload);
+    } else {
+      coinsToCreate.push(payload);
+    }
+  }
 
-    // store cryptocurrencies in the database
-    console.log(`Storing #${index + 1}/${availableCoins.length} ${coin.id}`);
+  // Create new coins
+  if (coinsToCreate.length > 0) {
+    console.log(`Creating ${coinsToCreate.length} coins...`);
+    await prismaClient.cryptocurrency.createMany({
+      data: coinsToCreate,
+    });
+  }
 
-    await prismaClient.cryptocurrency.upsert({
+  // Update existing coins
+  const updatePromises = coinsToUpdate.map((coin) => {
+    return prismaClient.cryptocurrency.update({
       where: {
-        coinId: coin.id,
+        coinId: coin.coinId,
+      },
+      data: {
+        currentPrice: parseFloat(coin.priceUsd || 0),
+        prices: coin.prices,
       },
-      update: payload,
-      create: payload,
     });
+  });
+
+  if (updatePromises.length > 0) {
+    console.log(`Updating ${coinsToUpdate.length} coins...`);
+    // show progress on update
+    for (const promise of updatePromises) {
+      const promiseIndex = updatePromises.indexOf(promise);
+      // sleep every 10 requests to avoid database failures
+      if (promiseIndex % 200 === 0 && promiseIndex !== 0) {
+        await sleep(10000); // sleep for 10 seconds
+      }
+
+      console.log(`Updating coin ${promiseIndex + 1}/${updatePromises.length}`);
+      await promise;
+    }
   }
+
+  console.log("Done!");
 }
 
 main();

scripts/sync_all_coins_description.js

@@ -53,10 +53,28 @@ const getAllAvailableTokens = async () => {
 async function main() {
   const availableCoins = await getAllAvailableTokens();
 
-  let discovered = 0;
-  for (let index = 0; index < availableCoins.length; index++) {
-    const coin = availableCoins[index];
+  const symbols = availableCoins.map((coin) => coin.SYMBOL);
+  const coinsFromDb = await prismaClient.cryptocurrency.findMany({
+    where: {
+      symbol: {
+        in: symbols,
+      },
+    },
+  });
+
+  const coinsToUpdate = availableCoins.filter((apiCoin) => {
+    const dbCoin = coinsFromDb.find((coin) => coin.symbol === apiCoin.SYMBOL);
+    return (
+      dbCoin &&
+      (dbCoin.description !== apiCoin.ASSET_DESCRIPTION ||
+        dbCoin.descriptionSummary !== apiCoin.ASSET_DESCRIPTION_SUMMARY)
+    );
+  });
 
+  console.log(`Updating: ${coinsToUpdate.length} coins`);
+  let discovered = 0;
+  for (let index = 0; index < coinsToUpdate.length; index++) {
+    const coin = coinsToUpdate[index];
     const {
       ASSET_DESCRIPTION,
       ASSET_DESCRIPTION_SUMMARY,
@@ -65,24 +83,9 @@ async function main() {
       SYMBOL,
     } = coin;
 
-    const findCoin = await prismaClient.cryptocurrency.findFirst({
-      where: {
-        symbol: SYMBOL,
-      },
-    });
-
-    if (!findCoin) {
-      console.log(
-        `#${index + 1}/${
-          availableCoins.length
-        } - Not found in the database, skipping...`
-      );
-      continue;
-    }
-
     console.log(
       `#${index + 1}/${
-        availableCoins.length
+        coinsToUpdate.length
       } - Updating coin ${NAME} (${SYMBOL})`
     );
 
@@ -98,10 +101,10 @@ async function main() {
     discovered += 1;
   }
 
-  // calculate the percentage of found tokens
+  // calculate the percentage of updated tokens
   const percentage = (discovered / availableCoins.length) * 100;
   console.log(
-    `Discovered ${discovered}/${
+    `Updated ${discovered}/${
       availableCoins.length
     } tokens (${percentage.toFixed(2)}%)`
   );

scripts/sync_available_coins.js

@@ -3,10 +3,16 @@ const { PrismaClient } = require("@prisma/client");
 const initSentry = require("./initSentry");
 initSentry();
 
+/**
+ * This script is used to sync the available coins from CoinGecko to Redis.
+ * This is used to populate the available coins in the frontend.
+ * This script is run on a cron job.
+ */
+
 /** @type {import('@prisma/client').PrismaClient} */
 const prismaClient = global.prisma || new PrismaClient();
 
-const mapCoinGeckoResponseToRedis = (coins) => {
+const mapCoinGeckoResponseToKeyValue = (coins) => {
   const redisCoins = coins.map((coin) => ({
     id: coin.coinId,
     coinId: coin.coinId,
@@ -20,10 +26,17 @@ const mapCoinGeckoResponseToRedis = (coins) => {
 
 async function main() {
   console.log("Fetching all available coins...");
-  const allCoins = await prismaClient.cryptocurrency.findMany();
+  const allCoins = await prismaClient.cryptocurrency.findMany({
+    select: {
+      coinId: true,
+      symbol: true,
+      name: true,
+      marketCapRank: true,
+    },
+  });
 
   console.log("Mapping coins to redis format...");
-  const mappedCoins = mapCoinGeckoResponseToRedis(allCoins);
+  const mappedCoins = mapCoinGeckoResponseToKeyValue(allCoins);
 
   console.log("Upserting coins to Prisma...");
   const key = "availableTokens";