fix: remove blocks cache displayed on /v2/blocks (#787)

* fix: remove blocks cache displayed on /v2/blocks

This was generating invalid results, because blocks were cached too much time.

It also provides very little performance benefit (if at all)

* test: remove blocks caching tests

config/config.exs

@@ -34,9 +34,7 @@ config :ae_mdw,
   sync: true,
   # email address where to send sync crash notification
   operators: [],
-  contract_cache_expiration_minutes: 1440,
-  # 5 days default generations cache expiration
-  generations_cache_expiration_minutes: 7200
+  contract_cache_expiration_minutes: 1440
 
 # Configures the endpoint
 config :ae_mdw, AeMdwWeb.Endpoint,

lib/ae_mdw/blocks.ex

@@ -8,7 +8,6 @@ defmodule AeMdw.Blocks do
   alias AeMdw.Db.Model
   alias AeMdw.Db.State
   alias AeMdw.Db.Util, as: DbUtil
-  alias AeMdw.EtsCache
   alias AeMdw.Database
   alias AeMdw.Util
   alias AeMdw.Validate
@@ -35,16 +34,6 @@ defmodule AeMdw.Blocks do
 
   @table Model.Block
 
-  @cache_table __MODULE__
-  @blocks_cache_threshold 6
-
-  @spec create_cache_table() :: :ok
-  def create_cache_table do
-    generations_cache_exp = Application.fetch_env!(:ae_mdw, :generations_cache_expiration_minutes)
-
-    EtsCache.new(@cache_table, generations_cache_exp, :ordered_set)
-  end
-
   @spec fetch_blocks(State.t(), direction(), range(), cursor() | nil, limit(), boolean()) ::
           {cursor() | nil, [block()], cursor() | nil}
   def fetch_blocks(state, direction, range, cursor, limit, sort_mbs?) do
@@ -59,7 +48,7 @@ defmodule AeMdw.Blocks do
 
     case Util.build_gen_pagination(cursor, direction, range, limit, last_gen) do
       {:ok, prev_cursor, range, next_cursor} ->
-        {serialize_cursor(prev_cursor), render_blocks(state, range, last_gen, sort_mbs?),
+        {serialize_cursor(prev_cursor), render_blocks(state, range, sort_mbs?),
          serialize_cursor(next_cursor)}
 
       :error ->
@@ -108,59 +97,24 @@ defmodule AeMdw.Blocks do
     end
   end
 
-  defp render_blocks(state, range, last_gen, sort_mbs?),
-    do: Enum.map(range, &render(state, &1, last_gen, sort_mbs?))
-
-  defp render(state, gen, last_gen, sort_mbs?) when gen > last_gen - @blocks_cache_threshold do
-    [key_block | micro_blocks] = fetch_gen_blocks(state, gen, last_gen)
-
-    put_mbs_from_db(key_block, micro_blocks, sort_mbs?)
-  end
-
-  defp render(state, gen, last_gen, sort_mbs?) do
-    [key_block | micro_blocks] = fetch_gen_blocks(state, gen, last_gen)
-
-    fetch_gen_from_cache(gen, key_block, micro_blocks, sort_mbs?)
-  end
-
-  defp fetch_gen_blocks(_state, last_gen, last_gen) do
-    # gets by height once the chain current generation might happen to be higher than last_gen in DB
-    {:ok, %{key_block: kb, micro_blocks: mbs}} =
-      :aec_chain.get_generation_by_height(last_gen, :forward)
-
-    ^last_gen = :aec_blocks.height(kb)
-
-    for block <- [kb | mbs] do
-      header = :aec_blocks.to_header(block)
-      :aec_headers.serialize_for_client(header, DbUtil.prev_block_type(header))
-    end
-  end
-
-  defp fetch_gen_blocks(state, gen, _last_gen) do
-    state
-    |> Collection.stream(@table, :backward, nil, {gen, <<>>})
-    |> Stream.take_while(&match?({^gen, _mb_index}, &1))
-    |> Enum.map(fn key -> State.fetch!(state, @table, key) end)
-    |> Enum.reverse()
-    |> Enum.map(fn Model.block(index: {_height, _mbi}, hash: hash) ->
-      header = :aec_db.get_header(hash)
-
-      :aec_headers.serialize_for_client(header, DbUtil.prev_block_type(header))
+  defp render_blocks(state, range, sort_mbs?) do
+    Enum.map(range, fn gen ->
+      [key_block | micro_blocks] =
+        state
+        |> Collection.stream(@table, :backward, nil, {gen, <<>>})
+        |> Stream.take_while(&match?({^gen, _mb_index}, &1))
+        |> Enum.map(fn key -> State.fetch!(state, @table, key) end)
+        |> Enum.reverse()
+        |> Enum.map(fn Model.block(index: {_height, _mbi}, hash: hash) ->
+          header = :aec_db.get_header(hash)
+
+          :aec_headers.serialize_for_client(header, DbUtil.prev_block_type(header))
+        end)
+
+      put_mbs_from_db(key_block, micro_blocks, sort_mbs?)
     end)
   end
 
-  defp fetch_gen_from_cache(gen, key_block, micro_blocks, sort_mbs?) do
-    case EtsCache.get(@cache_table, gen) do
-      {key_block, _indx} ->
-        key_block
-
-      nil ->
-        key_block = put_mbs_from_db(key_block, micro_blocks, sort_mbs?)
-        EtsCache.put(@cache_table, gen, key_block)
-        key_block
-    end
-  end
-
   defp put_mbs_from_db(key_block, micro_blocks, false) do
     micro_blocks =
       micro_blocks

lib/ae_mdw_web/supervisor.ex

@@ -1,15 +1,12 @@
 defmodule AeMdwWeb.Supervisor do
   use Supervisor
 
-  alias AeMdw.Blocks
-
   @spec start_link([]) :: {:ok, pid()}
   def start_link([]),
     do: Supervisor.start_link(__MODULE__, [], name: __MODULE__)
 
   @impl true
   def init([]) do
-    Blocks.create_cache_table()
     children = [AeMdwWeb.Endpoint]
 
     Supervisor.init(children, strategy: :one_for_one)

test/integration/ae_mdw_web/controllers/block_controller_test.exs

@@ -2,22 +2,18 @@ defmodule Integration.AeMdwWeb.BlockControllerTest do
   use AeMdwWeb.ConnCase, async: false
 
   alias :aeser_api_encoder, as: Enc
-  alias AeMdw.Blocks
   alias AeMdw.Validate
   alias AeMdw.Db.Model
   alias AeMdw.Db.State
   alias AeMdw.Db.Util, as: DbUtil
   alias AeMdwWeb.TestUtil
   alias AeMdw.Error.Input, as: ErrInput
-  alias AeMdw.EtsCache
 
   require Model
 
   @moduletag :integration
 
-  @blocks_table Blocks
   @default_limit 10
-  @blocks_cache_threshold 6
 
   describe "block" do
     test "get key block by hash", %{conn: conn} do
@@ -188,59 +184,6 @@ defmodule Integration.AeMdwWeb.BlockControllerTest do
       assert %{"data" => ^blocks} = conn |> get(prev_url) |> json_response(200)
     end
 
-    test "it gets uncached generations with range", %{conn: conn} do
-      state = State.new()
-      range_begin = DbUtil.last_gen(state) - @blocks_cache_threshold + 1
-      range_end = DbUtil.last_gen(state)
-      range = "#{range_begin}-#{range_end}"
-      limit = @blocks_cache_threshold
-      conn = get(conn, "/blocks/#{range}?limit=#{limit}")
-      response = json_response(conn, 200)
-
-      assert Enum.count(response["data"]) == limit
-
-      assert response["data"]
-             |> Enum.zip(range_begin..range_end)
-             |> Enum.all?(fn {%{"height" => height}, index} -> height == index end)
-
-      assert is_nil(EtsCache.get(@blocks_table, range_begin))
-      assert is_nil(EtsCache.get(@blocks_table, range_end))
-      assert is_nil(response["next"])
-    end
-
-    test "get a mix of uncached and cached generations with range", %{conn: conn} do
-      state = State.new()
-      remaining = 3
-      range_begin = DbUtil.last_gen(state) - @blocks_cache_threshold + 1 - remaining
-      range_end = DbUtil.last_gen(state)
-      range = "#{range_begin}-#{range_end}"
-      limit = @blocks_cache_threshold
-
-      conn = get(conn, "/blocks/#{range}?limit=#{limit}")
-      response = json_response(conn, 200)
-
-      assert Enum.count(response["data"]) == limit
-
-      assert response["data"]
-             |> Enum.zip(range_begin..range_end)
-             |> Enum.all?(fn {%{"height" => height}, index} -> height == index end)
-
-      assert {%{"height" => ^range_begin}, _insert_time} =
-               EtsCache.get(@blocks_table, range_begin)
-
-      assert is_nil(EtsCache.get(@blocks_table, range_end))
-      assert is_nil(EtsCache.get(@blocks_table, range_end - @blocks_cache_threshold + 1))
-
-      conn_next = get(conn, response["next"])
-      response_next = json_response(conn_next, 200)
-
-      assert Enum.count(response_next["data"]) == remaining
-
-      assert response_next["data"]
-             |> Enum.zip((range_begin + limit)..(range_begin + limit * 2 - 1))
-             |> Enum.all?(fn {%{"height" => height}, index} -> height == index end)
-    end
-
     test "get blocks and sorted microblocks in a single generation", %{conn: conn} do
       range = "471542-471542"
       count = 1