Store not-found values in stores to be able to skip strategies

.changeset/hot-gorillas-fly.md

@@ -0,0 +1,16 @@
+---
+'@3loop/transaction-decoder': minor
+---
+
+Refactor stores to call `set` when the data is not found in strategies. This introduces new types for store values, to
+be able to differentiate between a missing value, and one that was never requested.
+
+- 1.  `Success` - The data is found successfully in the store
+- 2.  `NotFound` - The data is found in the store, but is missing the value
+- 3.  `MetaEmpty` - The contract metadata is not found in the store
+
+This change requires the users of the library to persist the NotFound state. Having the NotFound state allows us
+to skip the strategy lookup, which is one of the most expensive operations when decoding transactions.
+
+We suggest to keep a timestamp for the NotFound state, and invalidate it after a certain period of time. This will
+ensure that the strategy lookup is not skipped indefinitely. Separately, users can upload their own data to the store.

apps/web/src/lib/contract-loader.ts

@@ -32,34 +32,28 @@ export const AbiStoreLive = Layer.succeed(
         }),
       ],
     },
-    set: ({ address = {} }) =>
+    set: (value) =>
       Effect.gen(function* () {
-        const addressMatches = Object.entries(address)
+        if (value.status !== 'success' || value.result.type !== 'address') {
+          // TODO: Store it to avoid fetching again
+          return
+        }
 
-        // Cache all addresses
-        yield* Effect.all(
-          addressMatches.map(([key, value]) =>
-            Effect.promise(() =>
-              prisma.contractAbi
-                .create({
-                  data: {
-                    address: key,
-                    abi: value,
-                  },
-                })
-                .catch((e) => {
-                  console.error('Failed to cache abi', e)
-                  return null
-                }),
-            ),
-          ),
-          {
-            concurrency: 'inherit',
-            batching: 'inherit',
-          },
+        yield* Effect.promise(() =>
+          prisma.contractAbi
+            .create({
+              data: {
+                address: value.result.address.toLowerCase(),
+                abi: value.result.abi,
+              },
+            })
+            .catch((e) => {
+              console.error('Failed to cache abi', e)
+              return null
+            }),
         )
       }),
-    get: ({ address }) =>
+    get: ({ address, chainID }) =>
       Effect.gen(function* () {
         const normAddress = address.toLowerCase()
         const cached = yield* Effect.promise(() =>
@@ -71,9 +65,21 @@ export const AbiStoreLive = Layer.succeed(
         )
 
         if (cached != null) {
-          return cached.abi
+          return {
+            status: 'success',
+            result: {
+              type: 'address',
+              address: address,
+              abi: cached.abi,
+              chainID: chainID,
+            },
+          }
+        }
+
+        return {
+          status: 'empty',
+          result: null,
         }
-        return null
       }),
   }),
 )
@@ -103,17 +109,32 @@ export const ContractMetaStoreLive = Layer.effect(
               }) as Promise<ContractData | null>,
           ).pipe(Effect.catchAll((_) => Effect.succeed(null)))
 
-          return data
+          if (data == null) {
+            return {
+              status: 'empty',
+              result: null,
+            }
+          } else {
+            return {
+              status: 'success',
+              result: data,
+            }
+          }
         }),
       set: ({ address, chainID }, contractMeta) =>
         Effect.gen(function* () {
           const normAddress = address.toLowerCase()
 
+          if (contractMeta.result == null) {
+            // TODO: Store it to avoid fetching again
+            return
+          }
+
           yield* Effect.tryPromise(() =>
             prisma.contractMeta.create({
               data: {
-                ...contractMeta,
-                decimals: contractMeta.decimals ?? 0,
+                ...contractMeta.result,
+                decimals: contractMeta.result.decimals ?? 0,
                 address: normAddress,
                 chainID: chainID,
               },

packages/transaction-decoder/README.md

@@ -33,33 +33,19 @@ const decoded = new TransactionDecoder({
         }
     },
     abiStore: {
-        get: async (req: {
-            chainID: number
-            address: string
-            event?: string | undefined
-            signature?: string | undefined
-        }) => {
+        get: async (req: GetAbiParams) => {
             return db.getContractAbi(req)
         },
-        set: async (req: {
-            address?: Record<string, string>
-            signature?: Record<string, string>
-        }) => {
+        set: async (req: ContractAbiResult) => {
             await db.setContractAbi(req)
         },
     },
     contractMetaStore: {
-        get: async (req: {
-            address: string
-            chainID: number
-        }) => {
+        get: async (req: ContractMetaParams) => {
             return db.getContractMeta(req)
         },
-        set: async (req: {
-            address: string
-            chainID: number
-        }) {
-            // NOTE: not yet called as we do not have any automatic resolve strategy
+        set: async (req: ContractMetaParams, val: ContractMetaResult) {
+            await db.setContractMeta(req, val)
         },
     },
 })
@@ -113,11 +99,11 @@ const AbiStoreLive = Layer.succeed(
   AbiStore,
   AbiStore.of({
     strategies: { default: [] },
-    set: ({ address = {}, func = {}, event = {} }) =>
+    set: (result: ContractAbiResult) =>
       Effect.sync(() => {
         // NOTE: Ignore caching as we relay only on local abis
       }),
-    get: ({ address, signature, event }) =>
+    get: ({ address, signature, event, chainID }) =>
       Effect.sync(() => {
         const signatureAbiMap = {
           '0x3593564c': 'execute(bytes,bytes[],uint256)',
@@ -131,10 +117,19 @@ const AbiStoreLive = Layer.succeed(
         const abi = signatureAbiMap[signature]
 
         if (abi) {
-          return abi
+          return {
+            type: 'func',
+            abi: `[${abi}]`,
+            address,
+            chainID: chainID,
+            signature,
+          }
         }
 
-        return null
+        return {
+          status: 'empty',
+          result: null,
+        }
       }),
   }),
 )
@@ -150,16 +145,19 @@ export const MetaStoreLive = Layer.succeed(
     ContractMetaStore.of({
         get: ({ address, chainID }) => Effect.sync(() => {
             return {
+              status: 'success',
+              result: {
                 address: request.address,
                 chainID: request.chainID,
                 contractName: 'Mock Contract',
                 contractAddress: request.address,
                 tokenSymbol: 'MOCK',
                 decimals: 18,
                 type: ContractType.ERC20,
+              },
             }
         }),
-        set: ({ address, chainID }) => Effect.sync(() => {
+        set: () => Effect.sync(() => {
             // NOTE: Ignore for now
         }),
     })

packages/transaction-decoder/src/abi-loader.ts

@@ -1,18 +1,35 @@
 import { Context, Effect, Either, RequestResolver, Request, Array, pipe } from 'effect'
 import { ContractABI, GetContractABIStrategy } from './abi-strategy/request-model.js'
 
+const STRATEGY_TIMEOUT = 5000
 export interface GetAbiParams {
   chainID: number
   address: string
   event?: string | undefined
   signature?: string | undefined
 }
 
-type ChainOrDefault = number | 'default'
+export interface ContractAbiSuccess {
+  status: 'success'
+  result: ContractABI
+}
+
+export interface ContractAbiNotFound {
+  status: 'not-found'
+  result: null
+}
+
+export interface ContractAbiEmpty {
+  status: 'empty'
+  result: null
+}
+
+export type ContractAbiResult = ContractAbiSuccess | ContractAbiNotFound | ContractAbiEmpty
 
-export interface AbiStore<Key = GetAbiParams, SetValue = ContractABI, Value = string | null> {
+type ChainOrDefault = number | 'default'
+export interface AbiStore<Key = GetAbiParams, Value = ContractAbiResult> {
   readonly strategies: Record<ChainOrDefault, readonly RequestResolver.RequestResolver<GetContractABIStrategy>[]>
-  readonly set: (value: SetValue) => Effect.Effect<void, never>
+  readonly set: (value: Value) => Effect.Effect<void, never>
   readonly get: (arg: Key) => Effect.Effect<Value, never>
   readonly getMany?: (arg: Array<Key>) => Effect.Effect<Array<Value>, never>
 }
@@ -50,36 +67,20 @@ const getMany = (requests: Array<GetAbiParams>) =>
     }
   })
 
-const setOnValue = (abi: ContractABI | null) =>
+const setValue = (abi: ContractABI | null) =>
   Effect.gen(function* () {
     const { set } = yield* AbiStore
-    // NOTE: Now we ignore the null value, but we might want to store it to avoid pinging the same strategy again?
-    if (abi) yield* set(abi)
+    yield* set(abi == null ? { status: 'not-found', result: null } : { status: 'success', result: abi })
   })
 
-const getBestMatch = (abi: ContractABI | null, request: AbiLoader) => {
+const getBestMatch = (abi: ContractABI | null) => {
   if (abi == null) return null
 
-  const { address, event, signature } = request
-
-  let result: string | null = null
-
-  const addressmatch = abi.address?.[address]
-  if (addressmatch != null) {
-    result = addressmatch
-  }
-
-  const funcmatch = signature ? abi.func?.[signature] : null
-  if (result == null && funcmatch != null) {
-    result = `[${funcmatch}]`
-  }
-
-  const eventmatch = event ? abi.event?.[event] : null
-  if (result == null && eventmatch != null) {
-    result = `[${eventmatch}]`
+  if (abi.type === 'address') {
+    return abi.abi
   }
 
-  return result
+  return `[${abi.abi}]`
 }
 
 /**
@@ -133,7 +134,9 @@ const AbiLoaderRequestResolver = RequestResolver.makeBatched((requests: Array<Ab
       getMany(uniqueRequests),
       Effect.map(
         Array.partitionMap((resp, i) => {
-          return resp == null ? Either.left(uniqueRequests[i]) : Either.right([uniqueRequests[i], resp] as const)
+          return resp.status === 'empty'
+            ? Either.left(uniqueRequests[i])
+            : Either.right([uniqueRequests[i], resp.result] as const)
         }),
       ),
       Effect.orElseSucceed(() => [uniqueRequests, []] as const),
@@ -144,7 +147,8 @@ const AbiLoaderRequestResolver = RequestResolver.makeBatched((requests: Array<Ab
       results,
       ([request, result]) => {
         const group = groups[makeKey(request)]
-        return Effect.forEach(group, (req) => Request.succeed(req, result), { discard: true })
+        const abi = result?.abi ?? null
+        return Effect.forEach(group, (req) => Request.succeed(req, abi), { discard: true })
       },
       {
         discard: true,
@@ -162,7 +166,9 @@ const AbiLoaderRequestResolver = RequestResolver.makeBatched((requests: Array<Ab
 
       const allAvailableStrategies = Array.prependAll(strategies.default, strategies[chainID] ?? [])
 
+      // TODO: Distinct the errors and missing data, so we can retry on errors
       return Effect.validateFirst(allAvailableStrategies, (strategy) => Effect.request(strategyRequest, strategy)).pipe(
+        Effect.timeout(STRATEGY_TIMEOUT),
         Effect.orElseSucceed(() => null),
       )
     })
@@ -172,12 +178,12 @@ const AbiLoaderRequestResolver = RequestResolver.makeBatched((requests: Array<Ab
       strategyResults,
       (abi, i) => {
         const request = remaining[i]
-        const result = getBestMatch(abi, request)
+        const result = getBestMatch(abi)
 
         const group = groups[makeKey(request)]
 
         return Effect.zipRight(
-          setOnValue(abi),
+          setValue(abi),
           Effect.forEach(group, (req) => Request.succeed(req, result), { discard: true }),
         )
       },
@@ -186,11 +192,12 @@ const AbiLoaderRequestResolver = RequestResolver.makeBatched((requests: Array<Ab
   }),
 ).pipe(RequestResolver.contextFromServices(AbiStore), Effect.withRequestCaching(true))
 
+// TODO: When failing to decode with one ABI, we should retry with other resolved ABIs
 export const getAndCacheAbi = (params: GetAbiParams) => {
   if (params.event === '0x' || params.signature === '0x') {
     return Effect.succeed(null)
   }
-  return Effect.withSpan(Effect.request(AbiLoader(params), AbiLoaderRequestResolver), 'GetAndCacheAbi', {
+  return Effect.withSpan(Effect.request(AbiLoader(params), AbiLoaderRequestResolver), 'AbiLoader.GetAndCacheAbi', {
     attributes: {
       chainID: params.chainID,
       address: params.address,

packages/transaction-decoder/src/abi-strategy/blockscout-abi.ts

@@ -4,7 +4,7 @@ import * as RequestModel from './request-model.js'
 async function fetchContractABI(
   { address, chainID }: RequestModel.GetContractABIStrategy,
   config: { apikey?: string; endpoint: string },
-) {
+): Promise<RequestModel.ContractABI> {
   const endpoint = config.endpoint
 
   const params: Record<string, string> = {
@@ -24,9 +24,10 @@ async function fetchContractABI(
 
   if (json.status === '1') {
     return {
-      address: {
-        [address]: json.result,
-      },
+      chainID,
+      address,
+      abi: json.result,
+      type: 'address',
     }
   }