Skip to content

Commit

Permalink
feat: Contract code on-demand fetcher (#9708)
Browse files Browse the repository at this point in the history
* Contract code on-demand fetcher

* Review comments #1

* Review comments #2: ignore addresses with nonce

* Review comments 3: fix threshold calculation

* Refine trigger_fetch function

* Skip updating retries number in case of failed eth_getCode response
  • Loading branch information
vbaranov authored and rimrakhimov committed May 10, 2024
1 parent f19d0be commit e271f25
Show file tree
Hide file tree
Showing 48 changed files with 527 additions and 75 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ defmodule BlockScoutWeb.AddressCoinBalanceController do
alias BlockScoutWeb.{AccessHelper, AddressCoinBalanceView, Controller}
alias Explorer.{Chain, Market}
alias Explorer.Chain.{Address, Wei}
alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand
alias Phoenix.View

def index(conn, %{"address_id" => address_hash_string, "type" => "JSON"} = params) do
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ defmodule BlockScoutWeb.AddressContractController do
alias BlockScoutWeb.AccessHelper
alias Explorer.{Chain, Market}
alias Explorer.SmartContract.Solidity.PublishHelper
alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand

def index(conn, %{"address_id" => address_hash_string} = params) do
address_options = [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ defmodule BlockScoutWeb.AddressController do
alias Explorer.{Chain, Market}
alias Explorer.Chain.Address.Counters
alias Explorer.Chain.{Address, Wei}
alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.ContractCode, as: ContractCodeOnDemand
alias Phoenix.View

def index(conn, %{"type" => "JSON"} = params) do
Expand Down Expand Up @@ -96,6 +97,8 @@ defmodule BlockScoutWeb.AddressController do
with {:ok, address_hash} <- Chain.string_to_address_hash(address_hash_string),
{:ok, address} <- Chain.hash_to_address(address_hash),
{:ok, false} <- AccessHelper.restricted_access?(address_hash_string, params) do
ContractCodeOnDemand.trigger_fetch(address)

render(
conn,
"_show_address_transactions.html",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ defmodule BlockScoutWeb.AddressDecompiledContractController do

alias BlockScoutWeb.AccessHelper
alias Explorer.{Chain, Market}
alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand

def index(conn, %{"address_id" => address_hash_string} = params) do
with {:ok, address_hash} <- Chain.string_to_address_hash(address_hash_string),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ defmodule BlockScoutWeb.AddressInternalTransactionController do
alias BlockScoutWeb.{AccessHelper, Controller, InternalTransactionView}
alias Explorer.{Chain, Market}
alias Explorer.Chain.{Address, Wei}
alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand
alias Phoenix.View

def index(conn, %{"address_id" => address_hash_string, "type" => "JSON"} = params) do
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ defmodule BlockScoutWeb.AddressLogsController do
alias BlockScoutWeb.{AccessHelper, AddressLogsView, Controller}
alias Explorer.{Chain, Market}
alias Explorer.Chain.Address
alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand
alias Phoenix.View

use BlockScoutWeb, :controller
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ defmodule BlockScoutWeb.AddressReadContractController do
alias Explorer.{Chain, Market}
alias Explorer.Chain.Address
alias Explorer.SmartContract.Reader
alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand

def index(conn, %{"address_id" => address_hash_string} = params) do
address_options = [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ defmodule BlockScoutWeb.AddressReadProxyController do
alias BlockScoutWeb.AccessHelper
alias Explorer.{Chain, Market}
alias Explorer.Chain.Address
alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand

def index(conn, %{"address_id" => address_hash_string} = params) do
address_options = [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ defmodule BlockScoutWeb.AddressTokenBalanceController do
alias BlockScoutWeb.AccessHelper
alias Explorer.Chain
alias Explorer.Chain.Address
alias Indexer.Fetcher.TokenBalanceOnDemand
alias Indexer.Fetcher.OnDemand.TokenBalance, as: TokenBalanceOnDemand

def index(conn, %{"address_id" => address_hash_string} = params) do
with true <- ajax?(conn),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ defmodule BlockScoutWeb.AddressTokenController do
alias BlockScoutWeb.{AccessHelper, AddressTokenView, Controller}
alias Explorer.{Chain, Market}
alias Explorer.Chain.Address
alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand
alias Phoenix.View

def index(conn, %{"address_id" => address_hash_string, "type" => "JSON"} = params) do
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ defmodule BlockScoutWeb.AddressTokenTransferController do
alias BlockScoutWeb.{AccessHelper, Controller, TransactionView}
alias Explorer.{Chain, Market}
alias Explorer.Chain.{Address, DenormalizationHelper}
alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand
alias Phoenix.View

import BlockScoutWeb.Chain,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ defmodule BlockScoutWeb.AddressTransactionController do

alias Explorer.Chain.{DenormalizationHelper, Transaction, Wei}

alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand
alias Phoenix.View

alias Plug.Conn
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ defmodule BlockScoutWeb.AddressValidationController do

alias BlockScoutWeb.{AccessHelper, BlockView, Controller}
alias Explorer.{Chain, Market}
alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand
alias Phoenix.View

def index(conn, %{"address_id" => address_hash_string, "type" => "JSON"} = params) do
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ defmodule BlockScoutWeb.AddressWithdrawalController do

alias Explorer.Chain.Wei

alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand
alias Phoenix.View

def index(conn, %{"address_id" => address_hash_string, "type" => "JSON"} = params) do
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ defmodule BlockScoutWeb.AddressWriteContractController do
alias BlockScoutWeb.AddressView
alias Explorer.{Chain, Market}
alias Explorer.Chain.Address
alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand

def index(conn, %{"address_id" => address_hash_string} = params) do
address_options = [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ defmodule BlockScoutWeb.AddressWriteProxyController do
alias BlockScoutWeb.{AccessHelper, AddressView}
alias Explorer.{Chain, Market}
alias Explorer.Chain.Address
alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand

def index(conn, %{"address_id" => address_hash_string} = params) do
address_options = [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ defmodule BlockScoutWeb.API.RPC.AddressController do
alias Explorer.{Chain, Etherscan}
alias Explorer.Chain.{Address, Wei}
alias Explorer.Etherscan.{Addresses, Blocks}
alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand

@api_true [api?: true]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,10 @@ defmodule BlockScoutWeb.API.V2.AddressController do
alias Explorer.Chain.{Address, Hash, Transaction}
alias Explorer.Chain.Address.Counters
alias Explorer.Chain.Token.Instance
alias Indexer.Fetcher.{CoinBalanceOnDemand, TokenBalanceOnDemand}

alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.ContractCode, as: ContractCodeOnDemand
alias Indexer.Fetcher.OnDemand.TokenBalance, as: TokenBalanceOnDemand

@transaction_necessity_by_association [
necessity_by_association: %{
Expand Down Expand Up @@ -86,6 +89,8 @@ defmodule BlockScoutWeb.API.V2.AddressController do
Address.maybe_preload_smart_contract_associations(address, @contract_address_preloads, @api_true) do
CoinBalanceOnDemand.trigger_fetch(fully_preloaded_address)

ContractCodeOnDemand.trigger_fetch(address)

conn
|> put_status(200)
|> render(:address, %{address: fully_preloaded_address |> maybe_preload_ens_to_address()})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ defmodule BlockScoutWeb.API.V2.TokenController do
alias BlockScoutWeb.API.V2.{AddressView, TransactionView}
alias Explorer.{Chain, Repo}
alias Explorer.Chain.{Address, BridgedToken, Token, Token.Instance}
alias Indexer.Fetcher.TokenTotalSupplyOnDemand
alias Indexer.Fetcher.OnDemand.TokenTotalSupply, as: TokenTotalSupplyOnDemand

import BlockScoutWeb.Chain,
only: [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ defmodule BlockScoutWeb.API.V2.TransactionController do
alias Explorer.Chain.PolygonZkevm.Reader, as: PolygonZkevmReader
alias Explorer.Chain.ZkSync.Reader, as: ZkSyncReader
alias Explorer.Counters.{FreshPendingTransactionsCounter, Transactions24hStats}
alias Indexer.Fetcher.FirstTraceOnDemand
alias Indexer.Fetcher.OnDemand.FirstTrace, as: FirstTraceOnDemand

action_fallback(BlockScoutWeb.API.V2.FallbackController)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ defmodule BlockScoutWeb.Tokens.HolderController do
alias BlockScoutWeb.Tokens.HolderView
alias Explorer.Chain
alias Explorer.Chain.Address
alias Indexer.Fetcher.TokenTotalSupplyOnDemand
alias Indexer.Fetcher.OnDemand.TokenTotalSupply, as: TokenTotalSupplyOnDemand
alias Phoenix.View

import BlockScoutWeb.Chain,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ defmodule BlockScoutWeb.Tokens.TransferController do
alias BlockScoutWeb.Tokens.TransferView
alias Explorer.Chain
alias Explorer.Chain.Address
alias Indexer.Fetcher.TokenTotalSupplyOnDemand
alias Indexer.Fetcher.OnDemand.TokenTotalSupply, as: TokenTotalSupplyOnDemand
alias Phoenix.View

import BlockScoutWeb.Chain, only: [split_list_by_page: 1, paging_options: 1, next_page_params: 3]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ defmodule BlockScoutWeb.TransactionRawTraceController do
alias BlockScoutWeb.{AccessHelper, TransactionController}
alias EthereumJSONRPC
alias Explorer.{Chain, Market}
alias Indexer.Fetcher.FirstTraceOnDemand
alias Indexer.Fetcher.OnDemand.FirstTrace, as: FirstTraceOnDemand

def index(conn, %{"transaction_id" => hash_string} = params) do
with {:ok, hash} <- Chain.string_to_transaction_hash(hash_string),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ defmodule BlockScoutWeb.Models.TransactionStateHelper do
alias Explorer.{Chain, PagingOptions}
alias Explorer.Chain.{Block, BlockNumberHelper, Transaction, Wei}
alias Explorer.Chain.Cache.StateChanges
alias Indexer.Fetcher.{CoinBalanceOnDemand, TokenBalanceOnDemand}
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.TokenBalance, as: TokenBalanceOnDemand

{:ok, burn_address_hash} = Chain.string_to_address_hash(burn_address_hash_string())
@burn_address_hash burn_address_hash
Expand Down
4 changes: 4 additions & 0 deletions apps/block_scout_web/lib/block_scout_web/notifier.ex
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,10 @@ defmodule BlockScoutWeb.Notifier do
Endpoint.broadcast("tokens:#{to_string(contract_address_hash)}", "token_total_supply", %{token: token})
end

def handle_event({:chain_event, :fetched_bytecode, :on_demand, [address_hash, fetched_bytecode]}) do
Endpoint.broadcast("addresses:#{to_string(address_hash)}", "fetched_bytecode", %{fetched_bytecode: fetched_bytecode})
end

def handle_event({:chain_event, :changed_bytecode, :on_demand, [address_hash]}) do
Endpoint.broadcast("addresses:#{to_string(address_hash)}", "changed_bytecode", %{})
end
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ defmodule BlockScoutWeb.RealtimeEventHandler do
Subscriber.to(:address_token_balances, :on_demand)
Subscriber.to(:token_total_supply, :on_demand)
Subscriber.to(:changed_bytecode, :on_demand)
Subscriber.to(:fetched_bytecode, :on_demand)
Subscriber.to(:eth_bytecode_db_lookup_started, :on_demand)
Subscriber.to(:zkevm_confirmed_batches, :realtime)
# Does not come from the indexer
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ defmodule BlockScoutWeb.Account.WatchlistView do
alias BlockScoutWeb.Account.WatchlistAddressView
alias Explorer.Account.WatchlistAddress
alias Explorer.Market
alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand

def coin_balance_status(address) do
CoinBalanceOnDemand.trigger_fetch(address)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ defmodule BlockScoutWeb.API.RPC.AddressControllerTest do
alias Explorer.Chain
alias Explorer.Chain.{Events.Subscriber, Transaction, Wei}
alias Explorer.Counters.{AddressesCounter, AverageBlockTime}
alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand
alias Explorer.Repo

setup :set_mox_global
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ defmodule BlockScoutWeb.API.RPC.EthControllerTest do

alias Explorer.Counters.{AddressesCounter, AverageBlockTime}
alias Explorer.Repo
alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand

@first_topic_hex_string_1 "0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65"
@first_topic_hex_string_2 "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1885,24 +1885,24 @@ defmodule BlockScoutWeb.API.V2.AddressControllerTest do
end
end

describe "checks TokenBalanceOnDemand" do
describe "checks Indexer.Fetcher.OnDemand.TokenBalance" do
setup do
Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.BlockNumber.child_id())
Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.BlockNumber.child_id())
old_env = Application.get_env(:indexer, Indexer.Fetcher.TokenBalanceOnDemand)
old_env = Application.get_env(:indexer, Indexer.Fetcher.OnDemand.TokenBalance)

Application.put_env(
:indexer,
Indexer.Fetcher.TokenBalanceOnDemand,
Indexer.Fetcher.OnDemand.TokenBalance,
Keyword.put(old_env, :fallback_threshold_in_blocks, 0)
)

on_exit(fn ->
Application.put_env(:indexer, Indexer.Fetcher.TokenBalanceOnDemand, old_env)
Application.put_env(:indexer, Indexer.Fetcher.OnDemand.TokenBalance, old_env)
end)
end

test "Indexer.Fetcher.TokenBalanceOnDemand broadcasts only updated balances", %{conn: conn} do
test "Indexer.Fetcher.OnDemand.TokenBalance broadcasts only updated balances", %{conn: conn} do
address = insert(:address)

ctbs_erc_20 =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ defmodule BlockScoutWeb.TransactionStateControllerTest do
alias Explorer.Chain.Wei
alias Indexer.Fetcher.CoinBalance.Catchup, as: CoinBalanceCatchup
alias Explorer.Counters.{AddressesCounter, AverageBlockTime}
alias Indexer.Fetcher.CoinBalanceOnDemand
alias Indexer.Fetcher.OnDemand.CoinBalance, as: CoinBalanceOnDemand

setup :set_mox_global

Expand Down
18 changes: 17 additions & 1 deletion apps/explorer/lib/explorer/chain/address.ex
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ defmodule Explorer.Chain.Address do

alias Ecto.Association.NotLoaded
alias Ecto.Changeset
alias Explorer.{Chain, PagingOptions}
alias Explorer.{Chain, PagingOptions, Repo}

alias Explorer.Chain.{
Address,
Expand Down Expand Up @@ -58,6 +58,8 @@ defmodule Explorer.Chain.Address do
:names
]}

@timeout :timer.minutes(1)

@typedoc """
* `fetched_coin_balance` - The last fetched balance from Nethermind
* `fetched_coin_balance_block_number` - the `t:Explorer.Chain.Block.t/0` `t:Explorer.Chain.Block.block_number/0` for
Expand Down Expand Up @@ -423,4 +425,18 @@ defmodule Explorer.Chain.Address do

Chain.select_repo(options).exists?(query)
end

@doc """
Sets contract_code for the given Explorer.Chain.Address
"""
@spec set_contract_code(Hash.Address.t(), binary()) :: {non_neg_integer(), nil}
def set_contract_code(address_hash, contract_code) when not is_nil(address_hash) and is_binary(contract_code) do
now = DateTime.utc_now()

Repo.update_all(
from(address in __MODULE__, where: address.hash == ^address_hash),
[set: [contract_code: contract_code, updated_at: now]],
timeout: @timeout
)
end
end
15 changes: 15 additions & 0 deletions apps/explorer/lib/explorer/chain/address/coin_balance.ex
Original file line number Diff line number Diff line change
Expand Up @@ -148,4 +148,19 @@ defmodule Explorer.Chain.Address.CoinBalance do
|> validate_required(@required_fields)
|> unique_constraint(:block_number, name: :address_coin_balances_address_hash_block_number_index)
end

@doc """
Query to fetch latest coin balance for the given address
"""
@spec latest_coin_balance_query(Hash.Address.t(), non_neg_integer() | {:error, :empty_database}) :: Ecto.Query.t()
def latest_coin_balance_query(address_hash, stale_balance_window) do
from(
cb in __MODULE__,
where: cb.address_hash == ^address_hash,
where: cb.block_number >= ^stale_balance_window,
where: is_nil(cb.value_fetched_at),
order_by: [desc: :block_number],
limit: 1
)
end
end

0 comments on commit e271f25

Please sign in to comment.