Feat/remove rebasing erc20

package.json

@@ -18,7 +18,8 @@
     "process:ogv": "sqd process:ogv",
     "process:mainnet": "sqd process:mainnet",
     "process:test": "sqd process:test",
-    "process": "sqd process"
+    "process": "sqd process",
+    "log:processing-times": "ts-node --require tsconfig-paths/register scripts/check-processing-times.ts"
   },
   "dependencies": {
     "@subsquid/archive-registry": "^3.3.2",

processing-times.log

@@ -0,0 +1,11 @@
+Processing Times
+==============================
+Version: v999
+Date: 2024-12-19T20:21:01.586Z
+==============================
+arbitrum: 53.74 minutes
+base: 96.13 minutes
+mainnet: 43.33 minutes
+oeth: 52.01 minutes
+ogv: 27.98 minutes
+ousd: 72.90 minutes

schema.graphql

@@ -325,6 +325,7 @@ type OToken @entity {
   totalSupply: BigInt!
   rebasingSupply: BigInt!
   nonRebasingSupply: BigInt!
+  holderCount: Int!
 }
 
 type OTokenAsset @entity {
@@ -346,6 +347,7 @@ type OTokenAddress @entity {
   earned: BigInt!
   credits: BigInt!
   delegatedTo: String
+  blockNumber: Int!
   lastUpdated: DateTime!
   history: [OTokenHistory!]! @derivedFrom(field: "address")
 }
@@ -651,7 +653,6 @@ type ERC20Holder @entity {
   address: String! @index
   account: String! @index
   balance: BigInt!
-  rebasingCredits: BigInt # Only used for rebasing tokens
 }
 
 type ERC20State @entity {
@@ -662,7 +663,6 @@ type ERC20State @entity {
   address: String!
   totalSupply: BigInt!
   holderCount: Int!
-  rebasingCreditsPerToken: BigInt # Only used for rebasing tokens
 }
 
 type ERC20Balance @entity {
@@ -673,7 +673,6 @@ type ERC20Balance @entity {
   address: String! @index
   account: String! @index
   balance: BigInt!
-  rebasingCredits: BigInt # Only used for rebasing tokens
 }
 
 type ERC20Transfer @entity {
@@ -693,6 +692,8 @@ type ProcessingStatus @entity {
   id: ID!
   timestamp: DateTime!
   blockNumber: Int!
+  startTimestamp: DateTime! # when we started processing
+  headTimestamp: DateTime # when we first hit head block
 }
 type AeroCLGaugeClaimFees @entity {
   id: ID!

scripts/check-processing-times.ts

@@ -0,0 +1,64 @@
+import fs from 'fs'
+
+const LIMIT = 1000
+
+const gql = (query: string) => query
+
+const executeQuery = async <T>(query: string): Promise<T> => {
+  const response = await fetch(`https://origin.squids.live/origin-squid@${process.argv[2]}/api/graphql`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({ query }),
+  })
+  const text = await response.text()
+  if (response.status !== 200) {
+    throw new Error(`Failed to execute query: ${text}`)
+  }
+  try {
+    return JSON.parse(text)
+  } catch (err) {
+    console.log(text)
+    throw err
+  }
+}
+
+const main = async () => {
+  const result = await executeQuery<{
+    data: {
+      processingStatuses: {
+        id: string
+        startTimestamp: string
+        headTimestamp: string
+        blockNumber: string
+        timestamp: string
+      }[]
+    }
+  }>(
+    gql(`
+    query MyQuery {
+      processingStatuses(limit: 10, orderBy: id_ASC) {
+        id
+        startTimestamp
+        headTimestamp
+        blockNumber
+        timestamp
+      }
+    }
+  `),
+  )
+
+  let output = `Processing Times\n`
+  output += '==============================\n'
+  output += `Version: ${process.argv[2]}\nDate: ${new Date().toISOString()}\n`
+  output += '==============================\n'
+  for (const status of result.data.processingStatuses) {
+    const minutes = (Date.parse(status.headTimestamp) - Date.parse(status.startTimestamp)) / (1000 * 60)
+    output += `${status.id}: ${minutes.toFixed(2)} minutes\n`
+  }
+  console.log(output)
+  fs.writeFileSync(`processing-times.log`, output)
+}
+
+main()

src/arbitrum/erc20.ts

@@ -1,7 +1,7 @@
-import { createERC20PollingTracker } from '@templates/erc20'
+import { createERC20EventTracker } from '@templates/erc20'
 import { WOETH_ARBITRUM_ADDRESS } from '@utils/addresses'
 
-export const arbitrumERC20s = createERC20PollingTracker({
+export const arbitrumERC20s = createERC20EventTracker({
   from: 178662968,
   address: WOETH_ARBITRUM_ADDRESS,
 })

src/base/bridged-woeth-strategy.ts

@@ -10,13 +10,14 @@ const filter = logFilter({
   topic0: [bridgedWOETHStrategyABI.events.WOETHPriceUpdated.topic],
 })
 export const bridgedWoethStrategy = {
+  name: 'bridged-woeth-strategy',
   from: 18689567,
   setup: (processor: EvmBatchProcessor) => {
     processor.addLog(filter.value)
   },
   process: async (ctx: Context) => {
     const result: EventWOETHPriceUpdated[] = []
-    for (const block of ctx.blocks) {
+    for (const block of ctx.blocksWithContent) {
       if (block.header.height < bridgedWoethStrategy.from) continue
       for (const log of block.logs) {
         if (filter.matches(log)) {

src/base/erc20.ts

@@ -1,43 +1,14 @@
-import * as otoken from '@abi/otoken'
 import { createERC20EventTracker } from '@templates/erc20/erc20-event'
 import { createERC20PollingTracker } from '@templates/erc20/erc20-polling'
-import { createRebasingERC20Tracker, getErc20RebasingParams } from '@templates/erc20/erc20-rebasing'
 import { OGN_BASE_ADDRESS } from '@utils/addresses'
 import { baseAddresses } from '@utils/addresses-base'
-import { logFilter } from '@utils/logFilter'
 
 export const baseERC20s = [
   // OGN
   createERC20EventTracker({
     from: 15676145,
     address: OGN_BASE_ADDRESS,
   }),
-  // superOETHb
-  createRebasingERC20Tracker({
-    from: 17819702,
-    address: baseAddresses.tokens.superOETHb,
-    rebasing: {
-      rebaseEventFilter: logFilter({
-        address: [baseAddresses.tokens.superOETHb],
-        topic0: [otoken.events.TotalSupplyUpdatedHighres.topic],
-        transaction: true,
-        range: { from: 17819702 },
-      }),
-      getCredits: async (ctx, block, address) => {
-        const oToken = new otoken.Contract(ctx, block.header, baseAddresses.tokens.superOETHb)
-        return oToken.creditsBalanceOfHighres(address).then((credits) => credits._0)
-      },
-      getCreditsPerToken: async (ctx, block) => {
-        const oToken = new otoken.Contract(ctx, block.header, baseAddresses.tokens.superOETHb)
-        return oToken.rebasingCreditsPerTokenHighres()
-      },
-      ...getErc20RebasingParams({
-        from: 17819702,
-        yieldDelegationFrom: 23192884,
-        address: baseAddresses.tokens.superOETHb,
-      }),
-    },
-  }),
   // wsuperOETHb
   createERC20EventTracker({
     from: 17819702,

src/base/exchange-rates.ts

@@ -1,7 +1,8 @@
 import { Context } from '@processor'
 import { ensureExchangeRates } from '@shared/post-processors/exchange-rates'
 import { EvmBatchProcessor } from '@subsquid/evm-processor'
-import { blockFrequencyUpdater } from '@utils/blockFrequencyUpdater'
+
+export const name = 'exchange-rates-base'
 
 export const from = 16586878
 
@@ -10,14 +11,14 @@ export const setup = (processor: EvmBatchProcessor) => {
 }
 
 export const process = async (ctx: Context) => {
-  const blockFrequencyUpdate = blockFrequencyUpdater({ from })
-  await blockFrequencyUpdate(ctx, async (ctx, block) => {
+  for (const block of ctx.frequencyBlocks) {
+    if (block.header.height < from) continue
     await ensureExchangeRates(ctx, block, [
       ['AERO', 'USD'],
       ['OGN', 'USD'],
       ['OGN', 'ETH'],
       ['ETH', 'USD'],
       ['superOETHb', 'USD'],
     ])
-  })
+  }
 }

src/base/strategies.ts

@@ -30,11 +30,23 @@ export const strategies: readonly IStrategyData[] = [
   },
 ]
 
-export const baseStrategies = strategies.map((s) => {
-  return defineProcessor({
-    name: s.name,
-    from: s.from,
-    setup: s.kind !== 'Vault' ? createStrategyRewardSetup(s) : createStrategySetup(s),
-    process: s.kind !== 'Vault' ? createStrategyRewardProcessor(s) : createStrategyProcessor(s),
-  })
-})
+export const baseStrategies = [
+  ...strategies.map((s) => {
+    return defineProcessor({
+      name: s.name,
+      from: s.from,
+      setup: createStrategySetup(s),
+      process: createStrategyProcessor(s),
+    })
+  }),
+  ...strategies
+    .filter((s) => s.kind !== 'Vault')
+    .map((s) => {
+      return defineProcessor({
+        name: s.name,
+        from: s.from,
+        setup: createStrategyRewardSetup(s),
+        process: createStrategyRewardProcessor(s),
+      })
+    }),
+]

src/base/super-oeth-b.ts

@@ -7,6 +7,8 @@ import { createOTokenWithdrawalsProcessor } from '@templates/withdrawals'
 import { aerodromePools, baseAddresses } from '@utils/addresses-base'
 
 const otokenProcessor = createOTokenProcessor({
+  name: 'Super OETHb',
+  symbol: 'superOETHb',
   from: 17819702,
   vaultFrom: 17819702,
   otokenAddress: baseAddresses.tokens.superOETHb,

src/base/validate.ts

@@ -1,5 +1,3 @@
-import assert from 'assert'
-
 import {
   ERC20Balance,
   OToken,
@@ -11,33 +9,29 @@ import {
   StrategyDailyYield,
 } from '@model'
 import { Context } from '@processor'
-import { env } from '@utils/env'
 import { entities } from '@validation/entities'
-import { validateExpectations } from '@validation/validate'
+import { validateBlocks } from '@validation/validate'
 
 export const name = 'validate-base'
 
-let firstBlock = true
+const strategyBalances = Object.keys(entities)
+  .filter((k) => k.startsWith('strategyBalances_superoethb_'))
+  .map((k) => entities[k as keyof typeof entities])
+const strategyDailyYields = Object.keys(entities)
+  .filter((k) => k.startsWith('strategyDailyYields_superoethb_'))
+  .map((k) => entities[k as keyof typeof entities])
+
+const expectationSets = [
+  { entity: OToken, expectations: entities.superoethb_oTokens },
+  { entity: OTokenAPY, expectations: entities.superoethb_oTokenApies },
+  { entity: OTokenHistory, expectations: entities.superoethb_oTokenHistories },
+  { entity: OTokenRebase, expectations: entities.superoethb_oTokenRebases },
+  { entity: OTokenDailyStat, expectations: entities.superoethb_oTokenDailyStats },
+  { entity: ERC20Balance, expectations: entities.superoethb_erc20Balances },
+  ...strategyBalances.map((entities) => ({ entity: StrategyBalance, expectations: entities })),
+  ...strategyDailyYields.map((entities) => ({ entity: StrategyDailyYield, expectations: entities })),
+]
 
 export const process = async (ctx: Context) => {
-  if (env.BLOCK_FROM || env.PROCESSOR) return
-  for (const block of ctx.blocks) {
-    await validateExpectations(ctx, block, OToken, firstBlock, entities.superoethb_oTokens)
-    await validateExpectations(ctx, block, OTokenAPY, firstBlock, entities.superoethb_oTokenApies)
-    await validateExpectations(ctx, block, OTokenHistory, firstBlock, entities.superoethb_oTokenHistories)
-    await validateExpectations(ctx, block, OTokenRebase, firstBlock, entities.superoethb_oTokenRebases)
-    await validateExpectations(ctx, block, OTokenDailyStat, firstBlock, entities.superoethb_oTokenDailyStats)
-    await validateExpectations(ctx, block, ERC20Balance, firstBlock, entities.superoethb_erc20Balances)
-    const strategyBalances = Object.keys(entities).filter((k) => k.startsWith('strategyBalances_superoethb_'))
-    assert(strategyBalances.length > 0, 'No strategyBalances found')
-    for (const key of strategyBalances) {
-      await validateExpectations(ctx, block, StrategyBalance, firstBlock, entities[key as keyof typeof entities])
-    }
-    const strategyDailyYields = Object.keys(entities).filter((k) => k.startsWith('strategyDailyYields_superoethb_'))
-    assert(strategyDailyYields.length > 0, 'No strategyDailyYields found')
-    for (const key of strategyDailyYields) {
-      await validateExpectations(ctx, block, StrategyDailyYield, firstBlock, entities[key as keyof typeof entities])
-    }
-    firstBlock = false
-  }
+  await validateBlocks(ctx, expectationSets)
 }