Skip to content

Commit

Permalink
Merge pull request #1458 from zmcNotafraid/issue-1454
Browse files Browse the repository at this point in the history
feat: all blocks api paginator query count is slowly
  • Loading branch information
zmcNotafraid committed Jun 14, 2023
2 parents d9ea4a2 + f506aea commit 0284c01
Show file tree
Hide file tree
Showing 6 changed files with 114 additions and 20 deletions.
2 changes: 2 additions & 0 deletions lib/godwoken_explorer/application.ex
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,9 @@ defmodule GodwokenExplorer.Application do
# web home api
GodwokenExplorer.Counters.AccountsCounter,
GodwokenExplorer.Counters.AverageBlockTime,
GodwokenExplorer.Chain.Cache.BlockCount,
GodwokenExplorer.Chain.Cache.TransactionCount,

# api
GodwokenExplorer.Chain.Cache.PolyVersion,
# admin
Expand Down
13 changes: 1 addition & 12 deletions lib/godwoken_explorer/chain.ex
Original file line number Diff line number Diff line change
Expand Up @@ -125,18 +125,7 @@ defmodule GodwokenExplorer.Chain do

@spec transaction_estimated_count() :: non_neg_integer()
def transaction_estimated_count do
cached_value = TransactionCount.get_count()

if is_nil(cached_value) do
%Postgrex.Result{rows: [[rows]]} =
Repo.query!(
"SELECT reltuples::BIGINT AS estimate FROM pg_class WHERE relname='transactions'"
)

trunc(rows)
else
cached_value
end
TransactionCount.estimated_count()
end

def home_api_data(blocks, txs) do
Expand Down
80 changes: 80 additions & 0 deletions lib/godwoken_explorer/chain/cache/block_count.ex
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
defmodule GodwokenExplorer.Chain.Cache.BlockCount do
@moduledoc """
Cache for estimated block count.
"""

@default_cache_period :timer.hours(2)

use GodwokenExplorer.Chain.MapCache,
name: :block_count,
key: :count,
key: :async_task,
global_ttl: cache_period(),
ttl_check_interval: :timer.minutes(15),
callback: &async_task_on_deletion(&1)

require Logger

alias GodwokenExplorer.{Repo, Block}

def estimated_count do
cached_value = __MODULE__.get_count()

if is_nil(cached_value) do
%Postgrex.Result{rows: [[count]]} =
Repo.query!(
"SELECT reltuples::BIGINT AS estimate FROM pg_class WHERE relname = 'blocks';"
)

max(count, 0)
else
cached_value
end
end

defp handle_fallback(:count) do
# This will get the task PID if one exists and launch a new task if not
# See next `handle_fallback` definition
get_async_task()

{:return, nil}
end

defp handle_fallback(:async_task) do
# If this gets called it means an async task was requested, but none exists
# so a new one needs to be launched
{:ok, task} =
Task.start(fn ->
try do
result = Repo.aggregate(Block, :count, :hash, timeout: :infinity)

set_count(result)
rescue
e ->
Logger.debug([
"Coudn't update block count test #{inspect(e)}"
])
end

set_async_task(nil)
end)

{:update, task}
end

# By setting this as a `callback` an async task will be started each time the
# `count` expires (unless there is one already running)
defp async_task_on_deletion({:delete, _, :count}), do: get_async_task()

defp async_task_on_deletion(_data), do: nil

defp cache_period do
"BLOCKS_COUNT_CACHE_PERIOD"
|> System.get_env("")
|> Integer.parse()
|> case do
{integer, ""} -> :timer.seconds(integer)
_ -> @default_cache_period
end
end
end
21 changes: 15 additions & 6 deletions lib/godwoken_explorer/chain/cache/transaction_count.ex
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,21 @@ defmodule GodwokenExplorer.Chain.Cache.TransactionCount do

alias GodwokenExplorer.{Repo, Transaction}

def estimated_count do
cached_value = __MODULE__.get_count()

if is_nil(cached_value) do
%Postgrex.Result{rows: [[count]]} =
Repo.query!(
"SELECT reltuples::BIGINT AS estimate FROM pg_class WHERE relname = 'transactions';"
)

max(count, 0)
else
cached_value
end
end

defp handle_fallback(:count) do
# This will get the task PID if one exists and launch a new task if not
# See next `handle_fallback` definition
Expand All @@ -32,12 +47,6 @@ defmodule GodwokenExplorer.Chain.Cache.TransactionCount do
Task.start(fn ->
try do
result = Repo.aggregate(Transaction, :count, :hash, timeout: :infinity)
# %Postgrex.Result{rows: [[rows]]} =
# Repo.query!(
# Repo.aggregate(:count, Transaction)
# )

# result = trunc(rows)

set_count(result)
rescue
Expand Down
16 changes: 15 additions & 1 deletion lib/godwoken_explorer_web/views/block_view.ex
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ defmodule GodwokenExplorer.BlockView do
import GodwokenRPC.Util, only: [utc_to_unix: 1]

alias GodwokenExplorer.{Block, Repo}
alias GodwokenExplorer.Chain.Cache.BlockCount

@block_capped_limit 500_000

def fields do
[
Expand Down Expand Up @@ -42,10 +45,21 @@ defmodule GodwokenExplorer.BlockView do
end

def list(page, page_size) do
paging_options =
if BlockCount.estimated_count() > @block_capped_limit do
[total_entries: @block_capped_limit]
else
[]
end

from(
b in Block,
order_by: [desc: :number]
)
|> Repo.paginate(page: page, page_size: page_size)
|> Repo.paginate(
page: page,
page_size: page_size,
options: paging_options
)
end
end
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ defmodule GodwokenExplorerWeb.API.HomeControllerTest do
describe "index" do
test "return home data", %{conn: conn, block: block, tx: tx, user: user, contract: contract} do
with_mock TransactionCount,
get_count: fn ->
estimated_count: fn ->
1
end do
conn =
Expand Down

0 comments on commit 0284c01

Please sign in to comment.