Added MerkleCache

lib/block_process.ex

@@ -73,7 +73,6 @@ defmodule BlockProcess do
     def with_block(block_hash, fun) do
       key = {__MODULE__, :block, block_hash}
 
-      # IO.inspect(Profiler.stacktrace())
       case Process.get(key, nil) do
         nil ->
           block =

lib/chain/state.ex

@@ -146,7 +146,7 @@ defmodule Chain.State do
               end
             end)
 
-          state_diff = MerkleTree.difference(Account.tree(acc_a), Account.tree(acc_b))
+          state_diff = MerkleCache.difference(Account.tree(acc_a), Account.tree(acc_b))
 
           if map_size(state_diff) > 0 do
             Map.merge(report, %{

lib/diode.ex

@@ -63,6 +63,7 @@ defmodule Diode do
 
     base_children = [
       worker(Stats, []),
+      worker(MerkleCache, []),
       supervisor(Model.Sql),
       supervisor(Channels),
       worker(Chain.BlockCache, [ets_extra]),

lib/mapmerkletree.ex

@@ -21,11 +21,11 @@ defmodule MapMerkleTree do
   end
 
   def merkle(tree) do
-    MerkleTree.copy(tree, MerkleTree2)
+    MerkleCache.merkle(tree)
   end
 
   def root_hash(tree) do
-    MerkleTree.root_hash(merkle(tree))
+    MerkleCache.root_hash(tree)
   end
 
   def root_hashes(tree) do

lib/merkle_cache.ex

@@ -0,0 +1,151 @@
+defmodule MerkleCache do
+  use GenServer
+  defstruct [:cache]
+
+  def start_link() do
+    GenServer.start_link(__MODULE__, [], name: __MODULE__)
+  end
+
+  @impl true
+  def init(_opts) do
+    {:ok, %MerkleCache{cache: []}}
+  end
+
+  def merkle(tree = {MapMerkleTree, _opts, dict}) do
+    if map_size(dict) > 1000 do
+      GenServer.call(__MODULE__, {:merkle, dict})
+    else
+      MerkleTree.copy(tree, MerkleTree2)
+    end
+  end
+
+  def root_hash(tree = {MapMerkleTree, _opts, dict}) do
+    if map_size(dict) > 1000 do
+      GenServer.call(__MODULE__, {:root_hash, dict})
+    else
+      merkle(tree) |> MerkleTree.root_hash()
+    end
+  end
+
+  def difference(tree1 = {MapMerkleTree, _opts, map1}, tree2 = {MapMerkleTree, _opts2, map2}) do
+    if MerkleTree.size(tree1) > 1000 do
+      GenServer.call(__MODULE__, {:difference, map1, map2})
+    else
+      MerkleTree.difference(tree1, tree2)
+    end
+  end
+
+  def difference(tree1 = {MapMerkleTree, _opts, map1}, tree2) do
+    if MerkleTree.size(tree1) > 1000 do
+      map2 = MerkleTree.to_list(tree2) |> Map.new()
+      GenServer.call(__MODULE__, {:difference, map1, map2})
+    else
+      MerkleTree.difference(tree1, tree2)
+    end
+  end
+
+  @impl true
+  def handle_call({:difference, map1, map2}, from, state) do
+    {:reply, tree1, state} = handle_call({:merkle, map1}, from, state)
+    root_hash = MerkleTree.root_hash(tree1)
+
+    {state, index} =
+      Enum.find_index(state.cache, fn %{root_hash: tree_root_hash} ->
+        tree_root_hash == root_hash
+      end)
+      |> case do
+        nil ->
+          {do_insert_new(state, MerkleTree.to_list(tree1)), 0}
+
+        index ->
+          {state, index}
+      end
+
+    {diff, state} = do_diff(state, map2, index)
+    {:reply, diff, state}
+  end
+
+  def handle_call({:root_hash, map}, from, state) do
+    {:reply, tree, state} = handle_call({:merkle, map}, from, state)
+    {:reply, MerkleTree.root_hash(tree), state}
+  end
+
+  def handle_call({:merkle, map}, _from, state = %{cache: cache}) do
+    tenPercent = div(map_size(map), 10)
+    lower = max(0, map_size(map) - tenPercent)
+    upper = map_size(map) + tenPercent
+
+    Enum.find_index(cache, fn %{tree: tree, sample: sample} ->
+      if lower < MerkleTree.size(tree) and MerkleTree.size(tree) < upper do
+        score =
+          Enum.reduce(sample, 0, fn key, score ->
+            if MerkleTree.get(tree, key) == Map.get(map, key) do
+              score + 1
+            else
+              score
+            end
+          end)
+
+        # IO.puts("score: #{score}")
+        score * 2 > length(sample)
+      end
+    end)
+    |> case do
+      nil ->
+        state = %{cache: [%{tree: tree} | _]} = do_insert_new(state, Map.to_list(map))
+        {:reply, tree, state}
+
+      index ->
+        {diff, state} = do_diff(state, map, index)
+        %{tree: tree} = Enum.at(cache, index)
+
+        tree2 =
+          Enum.reduce(diff, tree, fn {key, {_a, b}}, tree ->
+            if b == nil do
+              MerkleTree.delete(tree, key)
+            else
+              MerkleTree.insert(tree, key, b)
+            end
+          end)
+
+        {:reply, tree2, state}
+    end
+  end
+
+  defp do_insert_new(state = %{cache: cache}, items) when is_list(items) do
+    tree = MerkleTree2.new() |> MerkleTree.insert_items(items)
+    sample = Enum.shuffle(items) |> Enum.take(100) |> Enum.map(&elem(&1, 0))
+    selfdiff = %{}
+
+    new = %{
+      tree: tree,
+      sample: sample,
+      diffs: %{:erlang.phash2(selfdiff) => selfdiff},
+      root_hash: MerkleTree.root_hash(tree)
+    }
+
+    %{state | cache: [new | Enum.take(cache, 10)]}
+  end
+
+  defp do_diff(state = %{cache: cache}, map, index) do
+    {_time, {_key, diff, state}} =
+      :timer.tc(fn ->
+        %{tree: tree, diffs: diffs} = Enum.at(cache, index)
+        key = :erlang.phash2(map)
+
+        diff =
+          Map.get(diffs, key) ||
+            MerkleTree.difference(tree, MapMerkleTree.from_map(map))
+
+        cache =
+          List.update_at(cache, index, fn item ->
+            %{item | diffs: Map.put(diffs, key, diff)}
+          end)
+
+        {key, diff, %{state | cache: cache}}
+      end)
+
+    # IO.puts("MerkleCache.do_diff: #{div(time, 1000)}ms = #{key} / #{map_size(diff)}")
+    {diff, state}
+  end
+end

lib/merkletree.ex

@@ -23,6 +23,10 @@ defmodule MerkleTree do
     copy(merkle, mod)
   end
 
+  def copy(merkle = {mod, _, _}, mod) do
+    merkle
+  end
+
   def copy(merkle, mod) do
     insert_items(mod.new(), to_list(merkle))
   end

lib/remote_chain/edge.ex

@@ -152,11 +152,6 @@ defmodule RemoteChain.Edge do
           end)
         end
 
-        # Moonbeam.estimate_gas(Base16.encode(CallPermit.address()), Base16.encode(call))
-        # |> IO.inspect(label: "estimate_gas")
-        # {:error, %{"message" => error}} ->
-        #   error(error)
-
         tx =
           Shell.raw(CallPermit.wallet(), call,
             to: CallPermit.address(),

notes.exs

@@ -1,3 +1,54 @@
+# 10th May 2024
+
+bns = 0xaf60faa5cd840b724742f1af116168276112d6a6
+{acc, nr} = Chain.with_peak(fn block -> {Chain.Block.state(block) |> Chain.State.ensure_account(
+bns), Chain.Block.number(block)} end)
+File.write!("bns_account_#{nr}.etf", :erlang.term_to_binary(acc))
+
+Logger.configure(level: :warning)
+
+accs = ~w(bns_account_7144000.etf bns_account_7144224.etf bns_account_7144861.etf
+  bns_account_7145072.etf bns_account_7145706.etf) |> Enum.map(&File.read!(&1) |> :erlang.binary_to_term() |> Map.put(:root_hash, nil))
+
+check = fn acc ->
+  {time1, root1} = :timer.tc(fn -> MerkleTree.copy(Chain.Account.tree(acc), MerkleTree2) |> MerkleTree.root_hash() end)
+  {time2, root2} = :timer.tc(fn -> Chain.Account.root_hash(acc) end)
+  ^root1 = root2
+  {div(time1, 1000), div(time2, 1000), time1 / time2}
+end
+
+check2 = fn acc ->
+  {time1, root1} = :timer.tc(fn -> MerkleTree.copy(Chain.Account.tree(acc), MerkleTree2) |> MerkleTree.root_hash() end)
+  {time2, root2} = :timer.tc(fn -> MerkleCache.root_hash(Chain.Account.tree(acc)) end)
+  ^root1 = root2
+  {div(time1, 1000), div(time2, 1000), time1 / time2}
+end
+
+check_diff = fn acc, acc2 ->
+  {time1, diff1} = :timer.tc(fn -> MerkleTree.difference(Chain.Account.tree(acc), Chain.Account.tree(acc2)) end)
+  {time2, diff2} = :timer.tc(fn -> MerkleCache.difference(Chain.Account.tree(acc), Chain.Account.tree(acc2)) end)
+  ^diff1 = diff2
+  {div(time1, 1000), div(time2, 1000), time1 / time2}
+end
+
+:timer.tc(fn -> Chain.Account.root_hash(hd(accs)) end)
+:timer.tc(fn -> Chain.Account.root_hash(Enum.at(accs, 1)) end)
+
+check.(Enum.at(accs, 1))
+check2.(Enum.at(accs, 1))
+check_diff.(Enum.at(accs, 1), Enum.at(accs, 2))
+check_diff.(Enum.at(accs, 1), Enum.at(accs, 1))
+check_diff.(Enum.at(accs, 2), Enum.at(accs, 1))
+
+loop = fn loop, acc ->
+  Chain.Account.root_hash(acc)
+  loop.(loop, acc)
+end
+
+pid = spawn(fn -> loop.(loop, Enum.at(accs, 1)) end)
+spawn(fn -> Profiler.fprof(MerkleCache) end)
+
+
 # 9th May 2024
 
 node = :global.whereis_name({RemoteChain.NodeProxy, Chains.Moonbeam})