Reorganize queries and indexes for internal_transactions table

CHANGELOG.md

@@ -4,6 +4,8 @@
 
 ### Fixes
 
+- [2910](https://github.com/poanetwork/blockscout/pull/2910) - Reorganize queries and indexes for internal_transactions table
+
 ### Chore
 
 - [#2896](https://github.com/poanetwork/blockscout/pull/2896) - Disable Parity websockets tests

apps/explorer/lib/explorer/chain.ex

@@ -15,6 +15,7 @@ defmodule Explorer.Chain do
       preload: 2,
       select: 2,
       subquery: 1,
+      union: 2,
       union_all: 2,
       where: 2,
       where: 3
@@ -193,8 +194,50 @@ defmodule Explorer.Chain do
     direction = Keyword.get(options, :direction)
     paging_options = Keyword.get(options, :paging_options, @default_paging_options)
 
-    InternalTransaction
-    |> InternalTransaction.where_address_fields_match(hash, direction)
+    if direction == nil do
+      query_to_address_hash_wrapped =
+        InternalTransaction
+        |> InternalTransaction.where_address_fields_match(hash, :to_address_hash)
+        |> common_where_limit_order(paging_options)
+        |> wrapped_union_subquery()
+
+      query_from_address_hash_wrapped =
+        InternalTransaction
+        |> InternalTransaction.where_address_fields_match(hash, :from_address_hash)
+        |> common_where_limit_order(paging_options)
+        |> wrapped_union_subquery()
+
+      query_created_contract_address_hash_wrapped =
+        InternalTransaction
+        |> InternalTransaction.where_address_fields_match(hash, :created_contract_address_hash)
+        |> common_where_limit_order(paging_options)
+        |> wrapped_union_subquery()
+
+      query_to_address_hash_wrapped
+      |> union(^query_from_address_hash_wrapped)
+      |> union(^query_created_contract_address_hash_wrapped)
+      |> preload(transaction: :block)
+      |> join_associations(necessity_by_association)
+      |> Repo.all()
+    else
+      InternalTransaction
+      |> InternalTransaction.where_address_fields_match(hash, direction)
+      |> common_where_limit_order(paging_options)
+      |> preload(transaction: :block)
+      |> join_associations(necessity_by_association)
+      |> Repo.all()
+    end
+  end
+
+  def wrapped_union_subquery(query) do
+    from(
+      q in subquery(query),
+      select: q
+    )
+  end
+
+  defp common_where_limit_order(query, paging_options) do
+    query
     |> InternalTransaction.where_is_different_from_parent_transaction()
     |> InternalTransaction.where_block_number_is_not_null()
     |> page_internal_transaction(paging_options)
@@ -205,9 +248,6 @@ defmodule Explorer.Chain do
       desc: it.transaction_index,
       desc: it.index
     )
-    |> preload(transaction: :block)
-    |> join_associations(necessity_by_association)
-    |> Repo.all()
   end
 
   @doc """
@@ -2438,6 +2478,7 @@ defmodule Explorer.Chain do
     |> for_parent_transaction(hash)
     |> join_associations(necessity_by_association)
     |> where_transaction_has_multiple_internal_transactions()
+    |> InternalTransaction.where_is_different_from_parent_transaction()
     |> page_internal_transaction(paging_options)
     |> limit(^paging_options.page_size)
     |> order_by([internal_transaction], asc: internal_transaction.index)

apps/explorer/lib/explorer/chain/internal_transaction.ex

@@ -491,6 +491,18 @@ defmodule Explorer.Chain.InternalTransaction do
     )
   end
 
+  def where_address_fields_match(query, address_hash, :to_address_hash) do
+    where(query, [it], it.to_address_hash == ^address_hash)
+  end
+
+  def where_address_fields_match(query, address_hash, :from_address_hash) do
+    where(query, [it], it.from_address_hash == ^address_hash)
+  end
+
+  def where_address_fields_match(query, address_hash, :created_contract_address_hash) do
+    where(query, [it], it.created_contract_address_hash == ^address_hash)
+  end
+
   def where_is_different_from_parent_transaction(query) do
     where(
       query,

apps/explorer/lib/explorer/etherscan.ex

@@ -3,7 +3,7 @@ defmodule Explorer.Etherscan do
   The etherscan context.
   """
 
-  import Ecto.Query, only: [from: 2, where: 3, or_where: 3]
+  import Ecto.Query, only: [from: 2, where: 3, or_where: 3, union: 2]
 
   alias Explorer.Etherscan.Logs
   alias Explorer.{Chain, Repo}
@@ -97,6 +97,7 @@ defmodule Explorer.Etherscan do
 
     query
     |> Chain.where_transaction_has_multiple_internal_transactions()
+    |> InternalTransaction.where_is_different_from_parent_transaction()
     |> Repo.all()
   end
 
@@ -120,12 +121,29 @@ defmodule Explorer.Etherscan do
       ) do
     options = Map.merge(@default_options, raw_options)
 
+    direction =
+      case options do
+        %{filter_by: "to"} -> :to
+        %{filter_by: "from"} -> :from
+        _ -> nil
+      end
+
+    consensus_blocks =
+      from(
+        b in Block,
+        where: b.consensus == true
+      )
+
     query =
       from(
         it in InternalTransaction,
-        inner_join: t in assoc(it, :transaction),
-        inner_join: b in assoc(t, :block),
-        order_by: [{^options.order_by_direction, t.block_number}],
+        inner_join: b in subquery(consensus_blocks),
+        on: it.block_number == b.number,
+        order_by: [
+          {^options.order_by_direction, it.block_number},
+          {:desc, it.transaction_index},
+          {:desc, it.index}
+        ],
         limit: ^options.page_size,
         offset: ^offset(options),
         select:
@@ -135,12 +153,43 @@ defmodule Explorer.Etherscan do
           })
       )
 
-    query
-    |> Chain.where_transaction_has_multiple_internal_transactions()
-    |> where_address_match(address_hash, options)
-    |> where_start_block_match(options)
-    |> where_end_block_match(options)
-    |> Repo.all()
+    if direction == nil do
+      query_to_address_hash_wrapped =
+        query
+        |> InternalTransaction.where_address_fields_match(address_hash, :to_address_hash)
+        |> InternalTransaction.where_is_different_from_parent_transaction()
+        |> where_start_block_match(options)
+        |> where_end_block_match(options)
+        |> Chain.wrapped_union_subquery()
+
+      query_from_address_hash_wrapped =
+        query
+        |> InternalTransaction.where_address_fields_match(address_hash, :from_address_hash)
+        |> InternalTransaction.where_is_different_from_parent_transaction()
+        |> where_start_block_match(options)
+        |> where_end_block_match(options)
+        |> Chain.wrapped_union_subquery()
+
+      query_created_contract_address_hash_wrapped =
+        query
+        |> InternalTransaction.where_address_fields_match(address_hash, :created_contract_address_hash)
+        |> InternalTransaction.where_is_different_from_parent_transaction()
+        |> where_start_block_match(options)
+        |> where_end_block_match(options)
+        |> Chain.wrapped_union_subquery()
+
+      query_to_address_hash_wrapped
+      |> union(^query_from_address_hash_wrapped)
+      |> union(^query_created_contract_address_hash_wrapped)
+      |> Repo.all()
+    else
+      query
+      |> InternalTransaction.where_address_fields_match(address_hash, direction)
+      |> InternalTransaction.where_is_different_from_parent_transaction()
+      |> where_start_block_match(options)
+      |> where_end_block_match(options)
+      |> Repo.all()
+    end
   end
 
   @doc """

apps/explorer/priv/repo/migrations/20191203112646_internal_transactions_add_to_address_hash_index.exs

@@ -0,0 +1,25 @@
+defmodule Explorer.Repo.Migrations.InternalTransactionsAddToAddressHashIndex do
+  use Ecto.Migration
+
+  def change do
+    execute(
+      "CREATE INDEX IF NOT EXISTS internal_transactions_from_address_hash_partial_index on public.internal_transactions(from_address_hash) WHERE (((type = 'call') AND (index > 0)) OR (type != 'call')) AND (NOT (block_number IS NULL));"
+    )
+
+    execute(
+      "CREATE INDEX IF NOT EXISTS internal_transactions_to_address_hash_partial_index on public.internal_transactions(to_address_hash) WHERE (((type = 'call') AND (index > 0)) OR (type != 'call')) AND (NOT (block_number IS NULL));"
+    )
+
+    execute(
+      "CREATE INDEX IF NOT EXISTS internal_transactions_created_contract_address_hash_partial_index on public.internal_transactions(created_contract_address_hash) WHERE (((type = 'call') AND (index > 0)) OR (type != 'call')) AND (NOT (block_number IS NULL));"
+    )
+
+    drop_if_exists(
+      index(
+        :internal_transactions,
+        [:to_address_hash, :from_address_hash, :created_contract_address_hash, :type, :index],
+        name: "internal_transactions_to_address_hash_from_address_hash_created"
+      )
+    )
+  end
+end