Merge pull request #2733 from poanetwork/pp_uncles_cache

Add cache for first page of uncles

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## Current
 
 ### Features
+- [#2733](https://github.com/poanetwork/blockscout/pull/2733) - Add cache for first page of uncles
 - [#2735](https://github.com/poanetwork/blockscout/pull/2735) - Add pending transactions cache
 - [#2726](https://github.com/poanetwork/blockscout/pull/2726) - Remove internal_transaction block_number setting from blocks runner
 - [#2717](https://github.com/poanetwork/blockscout/pull/2717) - Improve speed of nonconsensus data removal

apps/block_scout_web/test/block_scout_web/controllers/block_controller_test.exs

@@ -5,6 +5,8 @@ defmodule BlockScoutWeb.BlockControllerTest do
   setup do
     Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Blocks.child_id())
     Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Blocks.child_id())
+    Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Uncles.child_id())
+    Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Uncles.child_id())
 
     :ok
   end

apps/block_scout_web/test/block_scout_web/controllers/chain_controller_test.exs

@@ -11,6 +11,8 @@ defmodule BlockScoutWeb.ChainControllerTest do
   setup do
     Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Blocks.child_id())
     Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Blocks.child_id())
+    Supervisor.terminate_child(Explorer.Supervisor, Explorer.Chain.Cache.Uncles.child_id())
+    Supervisor.restart_child(Explorer.Supervisor, Explorer.Chain.Cache.Uncles.child_id())
     start_supervised!(AddressesCounter)
     AddressesCounter.consolidate()
 

apps/explorer/config/config.exs

@@ -149,6 +149,10 @@ config :explorer, Explorer.Chain.Cache.PendingTransactions,
   ttl_check_interval: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(1), else: false),
   global_ttl: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(5))
 
+config :explorer, Explorer.Chain.Cache.Uncles,
+  ttl_check_interval: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(1), else: false),
+  global_ttl: if(System.get_env("DISABLE_INDEXER") == "true", do: :timer.seconds(5))
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

apps/explorer/lib/explorer/application.ex

@@ -15,7 +15,8 @@ defmodule Explorer.Application do
     NetVersion,
     PendingTransactions,
     TransactionCount,
-    Transactions
+    Transactions,
+    Uncles
   }
 
   alias Explorer.Chain.Supply.RSK
@@ -53,7 +54,8 @@ defmodule Explorer.Application do
       con_cache_child_spec(RSK.cache_name(), ttl_check_interval: :timer.minutes(1), global_ttl: :timer.minutes(30)),
       Transactions,
       Accounts,
-      PendingTransactions
+      PendingTransactions,
+      Uncles
     ]
 
     children = base_children ++ configurable_children()

apps/explorer/lib/explorer/chain.ex

@@ -55,7 +55,8 @@ defmodule Explorer.Chain do
     Blocks,
     PendingTransactions,
     TransactionCount,
-    Transactions
+    Transactions,
+    Uncles
   }
 
   alias Explorer.Chain.Import.Runner
@@ -1344,20 +1345,43 @@ defmodule Explorer.Chain do
     paging_options = Keyword.get(options, :paging_options) || @default_paging_options
     block_type = Keyword.get(options, :block_type, "Block")
 
-    if block_type == "Block" && !paging_options.key do
-      case Blocks.take_enough(paging_options.page_size) do
-        nil ->
-          elements = fetch_blocks(block_type, paging_options, necessity_by_association)
+    cond do
+      block_type == "Block" && !paging_options.key ->
+        block_from_cache(block_type, paging_options, necessity_by_association)
 
-          Blocks.update(elements)
+      block_type == "Uncle" && !paging_options.key ->
+        uncles_from_cache(block_type, paging_options, necessity_by_association)
 
-          elements
+      true ->
+        fetch_blocks(block_type, paging_options, necessity_by_association)
+    end
+  end
 
-        blocks ->
-          blocks
-      end
-    else
-      fetch_blocks(block_type, paging_options, necessity_by_association)
+  defp block_from_cache(block_type, paging_options, necessity_by_association) do
+    case Blocks.take_enough(paging_options.page_size) do
+      nil ->
+        elements = fetch_blocks(block_type, paging_options, necessity_by_association)
+
+        Blocks.update(elements)
+
+        elements
+
+      blocks ->
+        blocks
+    end
+  end
+
+  def uncles_from_cache(block_type, paging_options, necessity_by_association) do
+    case Uncles.take_enough(paging_options.page_size) do
+      nil ->
+        elements = fetch_blocks(block_type, paging_options, necessity_by_association)
+
+        Uncles.update(elements)
+
+        elements
+
+      blocks ->
+        blocks
     end
   end
 

apps/explorer/lib/explorer/chain/cache/uncles.ex

@@ -0,0 +1,48 @@
+defmodule Explorer.Chain.Cache.Uncles do
+  @moduledoc """
+  Caches the last known uncles
+  """
+
+  alias Explorer.Chain.Block
+  alias Explorer.Repo
+
+  use Explorer.Chain.OrderedCache,
+    name: :uncles,
+    max_size: 60,
+    ids_list_key: "uncle_numbers",
+    preload: :transactions,
+    preload: [miner: :names],
+    preload: :rewards,
+    preload: :nephews,
+    ttl_check_interval: Application.get_env(:explorer, __MODULE__)[:ttl_check_interval],
+    global_ttl: Application.get_env(:explorer, __MODULE__)[:global_ttl]
+
+  import Ecto.Query
+
+  @type element :: Block.t()
+
+  @type id :: non_neg_integer()
+
+  def element_to_id(%Block{number: number}), do: number
+
+  def update_from_second_degree_relations(second_degree_relations) when is_nil(second_degree_relations), do: :ok
+
+  def update_from_second_degree_relations(second_degree_relations) do
+    uncle_hashes =
+      second_degree_relations
+      |> Enum.map(& &1.uncle_hash)
+      |> Enum.uniq()
+
+    query =
+      from(
+        block in Block,
+        where: block.consensus == false and block.hash in ^uncle_hashes,
+        inner_join: nephews in assoc(block, :nephews),
+        preload: [nephews: block]
+      )
+
+    query
+    |> Repo.all()
+    |> update()
+  end
+end

apps/explorer/lib/explorer/chain/import/runner/blocks.ex

@@ -612,7 +612,8 @@ defmodule Explorer.Chain.Import.Runner.Blocks do
         b in Block.SecondDegreeRelation,
         join: s in subquery(query),
         on: b.nephew_hash == s.nephew_hash and b.uncle_hash == s.uncle_hash,
-        update: [set: [uncle_fetched_at: ^updated_at]]
+        update: [set: [uncle_fetched_at: ^updated_at]],
+        select: map(b, [:nephew_hash, :uncle_hash, :index])
       )
 
     try do

apps/explorer/test/explorer/chain/cache/uncles_test.exs

@@ -0,0 +1,28 @@
+defmodule Explorer.Chain.Cache.UnclesTest do
+  use Explorer.DataCase
+
+  alias Explorer.Chain.Cache.Uncles
+  alias Explorer.Repo
+
+  setup do
+    Supervisor.terminate_child(Explorer.Supervisor, Uncles.child_id())
+    Supervisor.restart_child(Explorer.Supervisor, Uncles.child_id())
+
+    :ok
+  end
+
+  describe "update_from_second_degree_relations/1" do
+    test "fetches an uncle from a second_degree_relation and adds it to the cache" do
+      block = insert(:block)
+      uncle = insert(:block, consensus: false)
+
+      uncle_hash = uncle.hash
+
+      second_degree_relation = insert(:block_second_degree_relation, uncle_hash: uncle_hash, nephew: block)
+
+      Uncles.update_from_second_degree_relations([second_degree_relation])
+
+      assert [%{hash: uncle_hash}] = Uncles.all()
+    end
+  end
+end

apps/indexer/lib/indexer/block/fetcher.ex

@@ -13,7 +13,7 @@ defmodule Indexer.Block.Fetcher do
   alias Explorer.Chain
   alias Explorer.Chain.{Address, Block, Hash, Import, Transaction}
   alias Explorer.Chain.Cache.Blocks, as: BlocksCache
-  alias Explorer.Chain.Cache.{Accounts, BlockNumber, PendingTransactions, Transactions}
+  alias Explorer.Chain.Cache.{Accounts, BlockNumber, PendingTransactions, Transactions, Uncles}
   alias Indexer.Block.Fetcher.Receipts
 
   alias Indexer.Fetcher.{
@@ -177,6 +177,7 @@ defmodule Indexer.Block.Fetcher do
       update_block_cache(inserted[:blocks])
       update_transactions_cache(inserted[:transactions], inserted[:fork_transactions])
       update_addresses_cache(inserted[:addresses])
+      update_uncles_cache(inserted[:block_second_degree_relations])
       result
     else
       {step, {:error, reason}} -> {:error, {step, reason}}
@@ -204,6 +205,10 @@ defmodule Indexer.Block.Fetcher do
 
   defp update_addresses_cache(addresses), do: Accounts.drop(addresses)
 
+  defp update_uncles_cache(updated_relations) do
+    Uncles.update_from_second_degree_relations(updated_relations)
+  end
+
   def import(
         %__MODULE__{broadcast: broadcast, callback_module: callback_module} = state,
         options

apps/indexer/lib/indexer/fetcher/uncle_block.ex

@@ -12,7 +12,7 @@ defmodule Indexer.Fetcher.UncleBlock do
   alias Ecto.Changeset
   alias EthereumJSONRPC.Blocks
   alias Explorer.Chain
-  alias Explorer.Chain.Cache.{Accounts, PendingTransactions}
+  alias Explorer.Chain.Cache.{Accounts, PendingTransactions, Uncles}
   alias Explorer.Chain.Hash
   alias Indexer.{Block, BufferedTask, Tracer}
   alias Indexer.Fetcher.UncleBlock
@@ -129,6 +129,7 @@ defmodule Indexer.Fetcher.UncleBlock do
          }) do
       {:ok, imported} ->
         Accounts.drop(imported[:addresses])
+        Uncles.update_from_second_degree_relations(imported[:block_second_degree_relations])
         retry(errors)
 
       {:error, {:import = step, [%Changeset{} | _] = changesets}} ->

apps/indexer/lib/indexer/temporary/uncles_without_index.ex

@@ -15,6 +15,7 @@ defmodule Indexer.Temporary.UnclesWithoutIndex do
   alias EthereumJSONRPC.Blocks
   alias Explorer.{Chain, Repo}
   alias Explorer.Chain.Block.SecondDegreeRelation
+  alias Explorer.Chain.Cache.Uncles
   alias Indexer.{BufferedTask, Tracer}
   alias Indexer.Fetcher.UncleBlock
 
@@ -99,6 +100,7 @@ defmodule Indexer.Temporary.UnclesWithoutIndex do
     case Chain.import(%{block_second_degree_relations: %{params: block_second_degree_relations_params}}) do
       {:ok, %{block_second_degree_relations: block_second_degree_relations}} ->
         UncleBlock.async_fetch_blocks(block_second_degree_relations)
+        Uncles.update_from_second_degree_relations(block_second_degree_relations)
 
         retry(errors)