Implement l2 transactions preprocessing

packages/backend/src/Application.ts

@@ -87,7 +87,6 @@ import { FeederGatewayClient } from './peripherals/starkware/FeederGatewayClient
 import { FetchClient } from './peripherals/starkware/FetchClient'
 import { handleServerError, reportError } from './tools/ErrorReporter'
 import { Logger } from './tools/Logger'
-import { shouldShowL2Transactions } from './utils/shouldShowL2Transactions'
 
 export class Application {
   start: () => Promise<void>
@@ -443,14 +442,15 @@ export class Application {
         preprocessedStateDetailsRepository,
         preprocessedAssetHistoryRepository,
         userTransactionRepository,
+        l2TransactionRepository,
         logger
       )
 
       const userStatisticsPreprocessor = new UserStatisticsPreprocessor(
         preprocessedUserStatisticsRepository,
         preprocessedAssetHistoryRepository,
-        preprocessedStateUpdateRepository,
         stateUpdateRepository,
+        l2TransactionRepository,
         kvStore,
         logger
       )
@@ -462,6 +462,7 @@ export class Application {
         perpetualHistoryPreprocessor,
         stateDetailsPreprocessor,
         userStatisticsPreprocessor,
+        l2TransactionRepository,
         logger,
         isPreprocessorEnabled
       )
@@ -479,14 +480,15 @@ export class Application {
         preprocessedStateDetailsRepository,
         preprocessedAssetHistoryRepository,
         userTransactionRepository,
+        l2TransactionRepository,
         logger
       )
 
       const userStatisticsPreprocessor = new UserStatisticsPreprocessor(
         preprocessedUserStatisticsRepository,
         preprocessedAssetHistoryRepository,
-        preprocessedStateUpdateRepository,
         stateUpdateRepository,
+        l2TransactionRepository,
         kvStore,
         logger
       )
@@ -498,6 +500,7 @@ export class Application {
         spotHistoryPreprocessor,
         stateDetailsPreprocessor,
         userStatisticsPreprocessor,
+        l2TransactionRepository,
         logger,
         isPreprocessorEnabled
       )
@@ -517,16 +520,14 @@ export class Application {
 
     // #endregion core
     // #region api
-    const showL2Transactions = shouldShowL2Transactions(config)
     const homeController = new HomeController(
       pageContextService,
       assetDetailsService,
       forcedTradeOfferViewService,
       userTransactionRepository,
       forcedTradeOfferRepository,
       l2TransactionRepository,
-      preprocessedStateDetailsRepository,
-      showL2Transactions
+      preprocessedStateDetailsRepository
     )
 
     const userController = new UserController(
@@ -541,8 +542,7 @@ export class Application {
       forcedTradeOfferViewService,
       withdrawableAssetRepository,
       preprocessedUserStatisticsRepository,
-      config.starkex.contracts.perpetual,
-      showL2Transactions
+      config.starkex.contracts.perpetual
     )
     const stateUpdateController = new StateUpdateController(
       pageContextService,
@@ -551,7 +551,7 @@ export class Application {
       userTransactionRepository,
       l2TransactionRepository,
       preprocessedAssetHistoryRepository,
-      showL2Transactions
+      preprocessedStateDetailsRepository
     )
     const transactionController = new TransactionController(
       pageContextService,
@@ -629,7 +629,6 @@ export class Application {
     this.start = async () => {
       logger.for(this).info('Starting')
 
-      await apiServer.listen()
       if (config.freshStart) await database.rollbackAll()
       await database.migrateToLatest()
       await preprocessor.catchUp()
@@ -641,6 +640,8 @@ export class Application {
       await stateUpdateWithBatchIdMigrator.migrate()
       await stateUpdater.initTree()
 
+      await apiServer.listen()
+
       if (config.enableSync) {
         transactionStatusService.start()
         await syncScheduler.start()

packages/backend/src/api/controllers/ForcedTradeOfferController.test.ts

@@ -54,6 +54,7 @@ describe(ForcedTradeOfferController.name, () => {
   const pageContext: PageContext = {
     user: undefined,
     tradingMode: 'perpetual',
+    showL2Transactions: true,
     chainId: 1,
     instanceName: 'dYdX',
     collateralAsset: fakeCollateralAsset,

packages/backend/src/api/controllers/HomeController.ts

@@ -12,6 +12,7 @@ import { PageContextService } from '../../core/PageContextService'
 import { PaginationOptions } from '../../model/PaginationOptions'
 import { ForcedTradeOfferRepository } from '../../peripherals/database/ForcedTradeOfferRepository'
 import { L2TransactionRepository } from '../../peripherals/database/L2TransactionRepository'
+import { sumUpTransactionCount } from '../../peripherals/database/PreprocessedL2TransactionsStatistics'
 import { PreprocessedStateDetailsRepository } from '../../peripherals/database/PreprocessedStateDetailsRepository'
 import { UserTransactionData } from '../../peripherals/database/transactions/UserTransaction'
 import { UserTransactionRepository } from '../../peripherals/database/transactions/UserTransactionRepository'
@@ -33,8 +34,7 @@ export class HomeController {
     private readonly userTransactionRepository: UserTransactionRepository,
     private readonly forcedTradeOfferRepository: ForcedTradeOfferRepository,
     private readonly l2TransactionRepository: L2TransactionRepository,
-    private readonly preprocessedStateDetailsRepository: PreprocessedStateDetailsRepository,
-    private readonly showL2Transactions: boolean
+    private readonly preprocessedStateDetailsRepository: PreprocessedStateDetailsRepository
   ) {}
 
   async getHomePage(
@@ -44,7 +44,7 @@ export class HomeController {
     const paginationOpts = { offset: 0, limit: 6 }
     const [
       l2Transactions,
-      l2TransactionsCount,
+      lastStateDetailsWithL2TransactionsStatistics,
       stateUpdates,
       stateUpdatesCount,
       forcedUserTransactions,
@@ -53,7 +53,7 @@ export class HomeController {
       availableOffersCount,
     ] = await Promise.all([
       this.l2TransactionRepository.getPaginatedWithoutMulti(paginationOpts),
-      this.l2TransactionRepository.countAllDistinctTransactionIds(),
+      this.preprocessedStateDetailsRepository.findLastWithL2TransactionsStatistics(),
       this.preprocessedStateDetailsRepository.getPaginated(paginationOpts),
       this.preprocessedStateDetailsRepository.countAll(),
       this.userTransactionRepository.getPaginated({
@@ -85,12 +85,10 @@ export class HomeController {
     const content = renderHomePage({
       context,
       tutorials: [], // explicitly no tutorials
-      l2Transactions: this.showL2Transactions
-        ? {
-            data: l2Transactions.map(l2TransactionToEntry),
-            total: l2TransactionsCount,
-          }
-        : undefined,
+      l2Transactions: l2Transactions.map(l2TransactionToEntry),
+      totalL2Transactions: sumUpTransactionCount(
+        lastStateDetailsWithL2TransactionsStatistics?.cumulativeL2TransactionsStatistics
+      ),
       stateUpdates: stateUpdateEntries,
       totalStateUpdates: stateUpdatesCount,
       forcedTransactions: forcedTransactionEntries,
@@ -111,16 +109,19 @@ export class HomeController {
   ): Promise<ControllerResult> {
     const context = await this.pageContextService.getPageContext(givenUser)
 
-    const [total, l2Transactions] = await Promise.all([
-      this.l2TransactionRepository.countAllDistinctTransactionIds(),
-      this.l2TransactionRepository.getPaginatedWithoutMulti(pagination),
-    ])
+    const [l2Transactions, lastStateDetailsWithL2TransactionsStatistics] =
+      await Promise.all([
+        this.l2TransactionRepository.getPaginatedWithoutMulti(pagination),
+        this.preprocessedStateDetailsRepository.findLastWithL2TransactionsStatistics(),
+      ])
 
     const content = renderHomeL2TransactionsPage({
       context,
       l2Transactions: l2Transactions.map(l2TransactionToEntry),
+      total: sumUpTransactionCount(
+        lastStateDetailsWithL2TransactionsStatistics?.cumulativeL2TransactionsStatistics
+      ),
       ...pagination,
-      total,
     })
     return { type: 'success', content }
   }

packages/backend/src/api/controllers/L2TransactionController.ts

@@ -27,7 +27,9 @@ export class L2TransactionController {
       return { type: 'not found' }
     }
     const aggregatedL2Transaction =
-      await this.l2TransactionRepository.findByTransactionId(transactionId)
+      await this.l2TransactionRepository.findAggregatedByTransactionId(
+        transactionId
+      )
 
     if (!aggregatedL2Transaction) {
       return {

packages/backend/src/api/controllers/StateUpdateController.ts

@@ -16,6 +16,8 @@ import {
   PreprocessedAssetHistoryRecord,
   PreprocessedAssetHistoryRepository,
 } from '../../peripherals/database/PreprocessedAssetHistoryRepository'
+import { sumUpTransactionCount } from '../../peripherals/database/PreprocessedL2TransactionsStatistics'
+import { PreprocessedStateDetailsRepository } from '../../peripherals/database/PreprocessedStateDetailsRepository'
 import { StateUpdateRepository } from '../../peripherals/database/StateUpdateRepository'
 import { UserTransactionData } from '../../peripherals/database/transactions/UserTransaction'
 import { UserTransactionRepository } from '../../peripherals/database/transactions/UserTransactionRepository'
@@ -38,7 +40,7 @@ export class StateUpdateController {
     private readonly userTransactionRepository: UserTransactionRepository,
     private readonly l2TransactionRepository: L2TransactionRepository,
     private readonly preprocessedAssetHistoryRepository: PreprocessedAssetHistoryRepository,
-    private readonly showL2Transactions: boolean
+    private readonly preprocessedStateDetailsRepository: PreprocessedStateDetailsRepository
   ) {}
 
   async getStateUpdatePage(
@@ -56,7 +58,7 @@ export class StateUpdateController {
       forcedUserTransactions,
       totalForcedUserTransactions,
       l2Transactions,
-      totalL2Transactions,
+      preprocessedStateDetails,
     ] = await Promise.all([
       this.stateUpdateRepository.findById(stateUpdateId),
       this.preprocessedAssetHistoryRepository.getByStateUpdateIdPaginated(
@@ -82,7 +84,7 @@ export class StateUpdateController {
           limit: 6,
         }
       ),
-      this.l2TransactionRepository.countAllDistinctTransactionIdsByStateUpdateId(
+      this.preprocessedStateDetailsRepository.findByStateUpdateId(
         stateUpdateId
       ),
     ])
@@ -127,12 +129,12 @@ export class StateUpdateController {
       balanceChanges: balanceChangeEntries,
       totalBalanceChanges,
       priceChanges: priceEntries,
-      l2Transactions: this.showL2Transactions
-        ? {
-            data: l2Transactions.map(l2TransactionToEntry),
-            total: totalL2Transactions,
-          }
-        : undefined,
+      l2Transactions: l2Transactions.map(l2TransactionToEntry),
+      totalL2Transactions: preprocessedStateDetails?.l2TransactionsStatistics
+        ? sumUpTransactionCount(
+            preprocessedStateDetails.l2TransactionsStatistics
+          )
+        : 'processing',
       transactions,
       totalTransactions: totalForcedUserTransactions,
     })
@@ -147,12 +149,12 @@ export class StateUpdateController {
   ): Promise<ControllerResult> {
     const context = await this.pageContextService.getPageContext(givenUser)
 
-    const [l2Transactions, total] = await Promise.all([
+    const [l2Transactions, preprocessedStateDetails] = await Promise.all([
       this.l2TransactionRepository.getPaginatedWithoutMultiByStateUpdateId(
         stateUpdateId,
         pagination
       ),
-      this.l2TransactionRepository.countAllDistinctTransactionIdsByStateUpdateId(
+      this.preprocessedStateDetailsRepository.findByStateUpdateId(
         stateUpdateId
       ),
     ])
@@ -162,7 +164,11 @@ export class StateUpdateController {
       id: stateUpdateId.toString(),
       l2Transactions: l2Transactions.map(l2TransactionToEntry),
       ...pagination,
-      total,
+      total: preprocessedStateDetails?.l2TransactionsStatistics
+        ? sumUpTransactionCount(
+            preprocessedStateDetails.l2TransactionsStatistics
+          )
+        : 'processing',
     })
 
     return { type: 'success', content }

packages/backend/src/api/controllers/UserController.ts

@@ -30,6 +30,7 @@ import {
   PreprocessedAssetHistoryRecord,
   PreprocessedAssetHistoryRepository,
 } from '../../peripherals/database/PreprocessedAssetHistoryRepository'
+import { sumUpTransactionCount } from '../../peripherals/database/PreprocessedL2TransactionsStatistics'
 import { PreprocessedUserStatisticsRepository } from '../../peripherals/database/PreprocessedUserStatisticsRepository'
 import {
   SentTransactionRecord,
@@ -60,8 +61,7 @@ export class UserController {
     private readonly forcedTradeOfferViewService: ForcedTradeOfferViewService,
     private readonly withdrawableAssetRepository: WithdrawableAssetRepository,
     private readonly preprocessedUserStatisticsRepository: PreprocessedUserStatisticsRepository,
-    private readonly exchangeAddress: EthereumAddress,
-    private readonly showL2Transactions: boolean
+    private readonly exchangeAddress: EthereumAddress
   ) {}
 
   async getUserRegisterPage(
@@ -126,7 +126,6 @@ export class UserController {
       userAssets,
       history,
       l2Transactions,
-      l2TransactionsCount,
       sentTransactions,
       userTransactions,
       userTransactionsCount,
@@ -150,7 +149,6 @@ export class UserController {
         starkKey,
         paginationOpts
       ),
-      this.l2TransactionRepository.countAllUserSpecific(starkKey),
       this.sentTransactionRepository.getByStarkKey(starkKey),
       this.userTransactionRepository.getByStarkKey(
         starkKey,
@@ -175,6 +173,16 @@ export class UserController {
       }
     }
 
+    let l2TransactionsStatistics = userStatistics.l2TransactionsStatistics
+    if (!l2TransactionsStatistics) {
+      const lastUserStatisticsWithL2TransactionsStatistics =
+        await this.preprocessedUserStatisticsRepository.findLastWithL2TransactionsStatisticsByStarkKey(
+          starkKey
+        )
+      l2TransactionsStatistics =
+        lastUserStatisticsWithL2TransactionsStatistics?.l2TransactionsStatistics
+    }
+
     const assetDetailsMap = await this.assetDetailsService.getAssetDetailsMap({
       userAssets: userAssets,
       assetHistory: history,
@@ -214,12 +222,8 @@ export class UserController {
       context,
       starkKey,
       ethereumAddress: registeredUser?.ethAddress,
-      l2Transactions: this.showL2Transactions
-        ? {
-            data: l2Transactions.map(l2TransactionToEntry),
-            total: l2TransactionsCount,
-          }
-        : undefined,
+      l2Transactions: l2Transactions.map(l2TransactionToEntry),
+      totalL2Transactions: sumUpTransactionCount(l2TransactionsStatistics),
       withdrawableAssets: withdrawableAssets.map((asset) => ({
         asset: {
           hashOrId:
@@ -316,19 +320,24 @@ export class UserController {
     pagination: PaginationOptions
   ): Promise<ControllerResult> {
     const context = await this.pageContextService.getPageContext(givenUser)
-    const [l2Transactions, l2TransactionsCount] = await Promise.all([
-      this.l2TransactionRepository.getUserSpecificPaginated(
-        starkKey,
-        pagination
-      ),
-      this.l2TransactionRepository.countAllUserSpecific(starkKey),
-    ])
+    const [l2Transactions, lastUserStatisticsWithL2TransactionsStatistics] =
+      await Promise.all([
+        this.l2TransactionRepository.getUserSpecificPaginated(
+          starkKey,
+          pagination
+        ),
+        this.preprocessedUserStatisticsRepository.findLastWithL2TransactionsStatisticsByStarkKey(
+          starkKey
+        ),
+      ])
 
     const content = renderUserL2TransactionsPage({
       context,
       starkKey,
       l2Transactions: l2Transactions.map(l2TransactionToEntry),
-      total: l2TransactionsCount,
+      total: sumUpTransactionCount(
+        lastUserStatisticsWithL2TransactionsStatistics?.l2TransactionsStatistics
+      ),
       ...pagination,
     })
     return { type: 'success', content }