[db] add num_transfers for evm_tokens

[analyzer] dead-reckon numTransfers

[api] update with evmtokens.numTransfers

linter

address comment

rename migration

analyzer/evmtokens/evm_tokens.go

@@ -70,6 +70,7 @@ type StaleToken struct {
 	Addr                  string
 	LastDownloadRound     *uint64
 	TotalSupply           common.BigInt
+	NumTransfers          uint64
 	Type                  *evm.EVMTokenType
 	AddrContextIdentifier string
 	AddrContextVersion    int
@@ -91,6 +92,7 @@ func (m main) getStaleTokens(ctx context.Context, limit int) ([]*StaleToken, err
 			&staleToken.Addr,
 			&staleToken.LastDownloadRound,
 			&totalSupply,
+			&staleToken.NumTransfers,
 			&staleToken.Type,
 			&staleToken.AddrContextIdentifier,
 			&staleToken.AddrContextVersion,
@@ -147,6 +149,7 @@ func (m main) processStaleToken(ctx context.Context, batch *storage.QueryBatch,
 			tokenData.Symbol,
 			tokenData.Decimals,
 			totalSupply,
+			staleToken.NumTransfers,
 		)
 	} else if *staleToken.Type != evm.EVMTokenTypeUnsupported {
 		mutable, err := evm.EVMDownloadMutatedToken(
@@ -161,7 +164,7 @@ func (m main) processStaleToken(ctx context.Context, batch *storage.QueryBatch,
 			return fmt.Errorf("downloading mutated token %s: %w", staleToken.Addr, err)
 		}
 		if mutable != nil && mutable.TotalSupply != nil {
-			batch.Queue(queries.RuntimeEVMTokenUpdate,
+			batch.Queue(queries.RuntimeEVMTokenTotalSupplyUpdate,
 				m.runtime,
 				staleToken.Addr,
 				mutable.TotalSupply.String(),

analyzer/queries/queries.go

@@ -451,6 +451,7 @@ var (
       token_analysis.token_address,
       token_analysis.last_download_round,
       token_analysis.total_supply,
+      token_analysis.num_transfers,
       evm_tokens.token_type,
       address_preimages.context_identifier,
       address_preimages.context_version,
@@ -479,16 +480,19 @@ var (
       )`
 
 	RuntimeEVMTokenAnalysisInsert = `
-    INSERT INTO analysis.evm_tokens (runtime, token_address, total_supply, last_mutate_round)
-      VALUES ($1, $2, $3, $4)
-    ON CONFLICT (runtime, token_address) DO NOTHING`
+    INSERT INTO analysis.evm_tokens (runtime, token_address, total_supply, num_transfers, last_mutate_round)
+      VALUES ($1, $2, $3, $4, $5)
+    ON CONFLICT (runtime, token_address) DO
+      UPDATE SET
+        num_transfers = analysis.evm_tokens.num_transfers + $4`
 
 	RuntimeEVMTokenAnalysisMutateUpsert = `
-    INSERT INTO analysis.evm_tokens (runtime, token_address, total_supply, last_mutate_round)
-      VALUES ($1, $2, $3, $4)
+    INSERT INTO analysis.evm_tokens (runtime, token_address, total_supply, num_transfers, last_mutate_round)
+      VALUES ($1, $2, $3, $4, $5)
     ON CONFLICT (runtime, token_address) DO
       UPDATE SET
         total_supply = analysis.evm_tokens.total_supply + $3,
+        num_transfers = analysis.evm_tokens.num_transfers + $4,
         last_mutate_round = excluded.last_mutate_round`
 
 	RuntimeEVMTokenAnalysisUpdate = `
@@ -500,21 +504,22 @@ var (
       token_address = $2`
 
 	RuntimeEVMTokenInsert = `
-    INSERT INTO chain.evm_tokens (runtime, token_address, token_type, token_name, symbol, decimals, total_supply)
-      VALUES ($1, $2, $3, $4, $5, $6, $7)`
+    INSERT INTO chain.evm_tokens (runtime, token_address, token_type, token_name, symbol, decimals, total_supply, num_transfers)
+      VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`
 
-	RuntimeEVMTokenUpdate = `
+	RuntimeEVMTokenTotalSupplyUpdate = `
     UPDATE chain.evm_tokens
     SET
       total_supply = $3
     WHERE
       runtime = $1 AND
       token_address = $2`
 
-	RuntimeEVMTokenTotalSupplyChangeUpdate = `
+	RuntimeEVMTokenDeltaUpdate = `
     UPDATE chain.evm_tokens
     SET
-      total_supply = total_supply + $3
+      total_supply = total_supply + $3,
+      num_transfers = num_transfers + $4
     WHERE
       runtime = $1 AND
       token_address = $2`

analyzer/runtime/evm/client.go

@@ -36,8 +36,9 @@ const (
 const NativeRuntimeTokenAddress = "oasis1runt1menat1vet0ken0000000000000000000000"
 
 type EVMPossibleToken struct {
-	Mutated           bool
-	TotalSupplyChange big.Int
+	Mutated            bool
+	TotalSupplyChange  big.Int
+	NumTransfersChange uint64
 }
 
 type EVMTokenData struct {

analyzer/runtime/extract.go

@@ -655,6 +655,8 @@ func extractEvents(blockData *BlockData, relatedAccountAddresses map[apiTypes.Ad
 					if _, ok := blockData.PossibleTokens[eventAddr]; !ok {
 						blockData.PossibleTokens[eventAddr] = &evm.EVMPossibleToken{}
 					}
+					// Mints, burns, and zero-value transfers all count as transfers.
+					blockData.PossibleTokens[eventAddr].NumTransfersChange++
 					// Mark as mutated if transfer is between zero address
 					// and nonzero address (either direction) and nonzero
 					// amount. These will change the total supply as mint/
@@ -753,6 +755,8 @@ func extractEvents(blockData *BlockData, relatedAccountAddresses map[apiTypes.Ad
 					if _, ok := blockData.PossibleTokens[eventAddr]; !ok {
 						blockData.PossibleTokens[eventAddr] = &evm.EVMPossibleToken{}
 					}
+					// Mints, burns, and zero-value transfers all count as transfers.
+					blockData.PossibleTokens[eventAddr].NumTransfersChange++
 					// Mark as mutated if transfer is between zero address
 					// and nonzero address (either direction) and nonzero
 					// amount. These will change the total supply as mint/

analyzer/runtime/runtime.go

@@ -297,20 +297,24 @@ func (m *processor) queueDbUpdates(batch *storage.QueryBatch, data *BlockData) {
 	// Insert EVM token addresses.
 	for addr, possibleToken := range data.PossibleTokens {
 		totalSupplyChange := possibleToken.TotalSupplyChange.String()
+		numTransfersChange := possibleToken.NumTransfersChange
 		if possibleToken.Mutated {
-			batch.Queue(queries.RuntimeEVMTokenAnalysisMutateUpsert, m.runtime, addr, totalSupplyChange, data.Header.Round)
+			batch.Queue(queries.RuntimeEVMTokenAnalysisMutateUpsert, m.runtime, addr, totalSupplyChange, numTransfersChange, data.Header.Round)
 		} else {
-			batch.Queue(queries.RuntimeEVMTokenAnalysisInsert, m.runtime, addr, totalSupplyChange, data.Header.Round)
+			batch.Queue(queries.RuntimeEVMTokenAnalysisInsert, m.runtime, addr, totalSupplyChange, numTransfersChange, data.Header.Round)
 		}
-		// Dead reckon total_supply because it's optional for ERC721 contracts.
+		// Dead reckon total_supply and num_transfers.
+		// Note that total_supply is optional for ERC721 contracts.
 		// If the evm_tokens analyzer is able to fetch the total supply from the node,
-		// it will supersede this.
-		if possibleToken.TotalSupplyChange.Cmp(&big.Int{}) != 0 {
+		// it will supersede this, but implementing totalSupply() is optional for ERC721 contracts,
+		// so we have to maintain this dead-reckoned fallback.
+		if numTransfersChange != 0 || possibleToken.TotalSupplyChange.Cmp(&big.Int{}) != 0 {
 			batch.Queue(
-				queries.RuntimeEVMTokenTotalSupplyChangeUpdate,
+				queries.RuntimeEVMTokenDeltaUpdate,
 				m.runtime,
 				addr,
 				totalSupplyChange,
+				numTransfersChange,
 			)
 		}
 	}

api/spec/v1.yaml

@@ -2537,6 +2537,11 @@ components:
         total_supply:
           <<: *BigIntType
           description: The total number of base units available.
+        num_transfers:
+          type: integer
+          format: int64
+          description: |
+            The total number of transfers of this token.
         num_holders:
           type: integer
           format: int64

storage/client/client.go

@@ -1498,6 +1498,7 @@ func (c *StorageClient) RuntimeTokens(ctx context.Context, p apiTypes.GetRuntime
 			&t.Symbol,
 			&t.Decimals,
 			&t.TotalSupply,
+			&t.NumTransfers,
 			&t.Type,
 			&t.NumHolders,
 			&t.IsVerified,

storage/client/queries/queries.go

@@ -482,6 +482,7 @@ const (
 			tokens.symbol,
 			tokens.decimals,
 			tokens.total_supply,
+			tokens.num_transfers,
 			CASE -- NOTE: There are three queries that use this CASE via copy-paste; edit both if changing.
 				WHEN tokens.token_type = 20 THEN 'ERC20'
 				WHEN tokens.token_type = 721 THEN 'ERC721'

storage/migrations/14_evm_token_transfers.up.sql

@@ -0,0 +1,44 @@
+BEGIN;
+
+ALTER TABLE chain.evm_tokens ADD COLUMN num_transfers UINT63 NOT NULL DEFAULT 0;
+ALTER TABLE analysis.evm_tokens ADD COLUMN num_transfers UINT63 NOT NULL DEFAULT 0;
+
+-- Backfill chain.evm_tokens.num_transfers
+WITH transfers AS (
+    SELECT runtime, DECODE(body ->> 'address', 'base64') AS eth_addr, COUNT(*) AS num_xfers
+        FROM chain.runtime_events
+        GROUP BY runtime, eth_addr
+)
+UPDATE chain.evm_tokens as tokens
+    SET num_transfers = transfers.num_xfers
+    FROM transfers
+    LEFT JOIN chain.address_preimages as preimages 
+    ON
+        preimages.address_data = transfers.eth_addr AND
+        preimages.context_identifier = 'oasis-runtime-sdk/address: secp256k1eth' AND
+        preimages.context_version = 0
+    WHERE
+        tokens.runtime = transfers.runtime AND
+        tokens.token_address = preimages.address;
+
+-- Backfill analysis.evm_tokens.num_transfers for tokens that haven't been processed yet.
+-- These values will be inserted into chain.evm_tokens when the token is downloaded.
+WITH transfers AS (
+    SELECT runtime, DECODE(body ->> 'address', 'base64') AS eth_addr, COUNT(*) AS num_xfers
+        FROM chain.runtime_events
+        GROUP BY runtime, eth_addr
+)
+UPDATE analysis.evm_tokens as tokens
+    SET num_transfers = transfers.num_xfers
+    FROM transfers
+    LEFT JOIN chain.address_preimages as preimages 
+    ON
+        preimages.address_data = transfers.eth_addr AND
+        preimages.context_identifier = 'oasis-runtime-sdk/address: secp256k1eth' AND
+        preimages.context_version = 0
+    WHERE
+        tokens.runtime = transfers.runtime AND
+        tokens.token_address = preimages.address AND
+        tokens.last_download_round IS NULL;
+
+COMMIT;