Merge pull request #1458 from zmcNotafraid/issue-1454

feat: all blocks api paginator query count is slowly

lib/godwoken_explorer/application.ex

@@ -38,7 +38,9 @@ defmodule GodwokenExplorer.Application do
           # web home api
           GodwokenExplorer.Counters.AccountsCounter,
           GodwokenExplorer.Counters.AverageBlockTime,
+          GodwokenExplorer.Chain.Cache.BlockCount,
           GodwokenExplorer.Chain.Cache.TransactionCount,
+
           # api
           GodwokenExplorer.Chain.Cache.PolyVersion,
           # admin

lib/godwoken_explorer/chain.ex

@@ -125,18 +125,7 @@ defmodule GodwokenExplorer.Chain do
 
   @spec transaction_estimated_count() :: non_neg_integer()
   def transaction_estimated_count do
-    cached_value = TransactionCount.get_count()
-
-    if is_nil(cached_value) do
-      %Postgrex.Result{rows: [[rows]]} =
-        Repo.query!(
-          "SELECT reltuples::BIGINT AS estimate FROM pg_class WHERE relname='transactions'"
-        )
-
-      trunc(rows)
-    else
-      cached_value
-    end
+    TransactionCount.estimated_count()
   end
 
   def home_api_data(blocks, txs) do

lib/godwoken_explorer/chain/cache/block_count.ex

@@ -0,0 +1,80 @@
+defmodule GodwokenExplorer.Chain.Cache.BlockCount do
+  @moduledoc """
+  Cache for estimated block count.
+  """
+
+  @default_cache_period :timer.hours(2)
+
+  use GodwokenExplorer.Chain.MapCache,
+    name: :block_count,
+    key: :count,
+    key: :async_task,
+    global_ttl: cache_period(),
+    ttl_check_interval: :timer.minutes(15),
+    callback: &async_task_on_deletion(&1)
+
+  require Logger
+
+  alias GodwokenExplorer.{Repo, Block}
+
+  def estimated_count do
+    cached_value = __MODULE__.get_count()
+
+    if is_nil(cached_value) do
+      %Postgrex.Result{rows: [[count]]} =
+        Repo.query!(
+          "SELECT reltuples::BIGINT AS estimate FROM pg_class WHERE relname = 'blocks';"
+        )
+
+      max(count, 0)
+    else
+      cached_value
+    end
+  end
+
+  defp handle_fallback(:count) do
+    # This will get the task PID if one exists and launch a new task if not
+    # See next `handle_fallback` definition
+    get_async_task()
+
+    {:return, nil}
+  end
+
+  defp handle_fallback(:async_task) do
+    # If this gets called it means an async task was requested, but none exists
+    # so a new one needs to be launched
+    {:ok, task} =
+      Task.start(fn ->
+        try do
+          result = Repo.aggregate(Block, :count, :hash, timeout: :infinity)
+
+          set_count(result)
+        rescue
+          e ->
+            Logger.debug([
+              "Coudn't update block count test #{inspect(e)}"
+            ])
+        end
+
+        set_async_task(nil)
+      end)
+
+    {:update, task}
+  end
+
+  # By setting this as a `callback` an async task will be started each time the
+  # `count` expires (unless there is one already running)
+  defp async_task_on_deletion({:delete, _, :count}), do: get_async_task()
+
+  defp async_task_on_deletion(_data), do: nil
+
+  defp cache_period do
+    "BLOCKS_COUNT_CACHE_PERIOD"
+    |> System.get_env("")
+    |> Integer.parse()
+    |> case do
+      {integer, ""} -> :timer.seconds(integer)
+      _ -> @default_cache_period
+    end
+  end
+end

lib/godwoken_explorer/chain/cache/transaction_count.ex

@@ -17,6 +17,21 @@ defmodule GodwokenExplorer.Chain.Cache.TransactionCount do
 
   alias GodwokenExplorer.{Repo, Transaction}
 
+  def estimated_count do
+    cached_value = __MODULE__.get_count()
+
+    if is_nil(cached_value) do
+      %Postgrex.Result{rows: [[count]]} =
+        Repo.query!(
+          "SELECT reltuples::BIGINT AS estimate FROM pg_class WHERE relname = 'transactions';"
+        )
+
+      max(count, 0)
+    else
+      cached_value
+    end
+  end
+
   defp handle_fallback(:count) do
     # This will get the task PID if one exists and launch a new task if not
     # See next `handle_fallback` definition
@@ -32,12 +47,6 @@ defmodule GodwokenExplorer.Chain.Cache.TransactionCount do
       Task.start(fn ->
         try do
           result = Repo.aggregate(Transaction, :count, :hash, timeout: :infinity)
-          # %Postgrex.Result{rows: [[rows]]} =
-          #   Repo.query!(
-          #     Repo.aggregate(:count, Transaction)
-          #   )
-
-          # result = trunc(rows)
 
           set_count(result)
         rescue

lib/godwoken_explorer_web/views/block_view.ex

@@ -5,6 +5,9 @@ defmodule GodwokenExplorer.BlockView do
   import GodwokenRPC.Util, only: [utc_to_unix: 1]
 
   alias GodwokenExplorer.{Block, Repo}
+  alias GodwokenExplorer.Chain.Cache.BlockCount
+
+  @block_capped_limit 500_000
 
   def fields do
     [
@@ -42,10 +45,21 @@ defmodule GodwokenExplorer.BlockView do
   end
 
   def list(page, page_size) do
+    paging_options =
+      if BlockCount.estimated_count() > @block_capped_limit do
+        [total_entries: @block_capped_limit]
+      else
+        []
+      end
+
     from(
       b in Block,
       order_by: [desc: :number]
     )
-    |> Repo.paginate(page: page, page_size: page_size)
+    |> Repo.paginate(
+      page: page,
+      page_size: page_size,
+      options: paging_options
+    )
   end
 end

test/godwoken_explorer_web/controllers/api/home_controller_test.exs

@@ -22,7 +22,7 @@ defmodule GodwokenExplorerWeb.API.HomeControllerTest do
   describe "index" do
     test "return home data", %{conn: conn, block: block, tx: tx, user: user, contract: contract} do
       with_mock TransactionCount,
-        get_count: fn ->
+        estimated_count: fn ->
           1
         end do
         conn =