From afb61c2f7bb0a6cf11c7844fc17032982eb4102e Mon Sep 17 00:00:00 2001 From: Dominic Letz Date: Fri, 10 May 2024 17:58:33 +0200 Subject: [PATCH] Added MerkleCache --- lib/block_process.ex | 1 - lib/chain/state.ex | 2 +- lib/diode.ex | 1 + lib/mapmerkletree.ex | 4 +- lib/merkle_cache.ex | 151 +++++++++++++++++++++++++++++++++++++++ lib/merkletree.ex | 4 ++ lib/remote_chain/edge.ex | 5 -- notes.exs | 51 +++++++++++++ 8 files changed, 210 insertions(+), 9 deletions(-) create mode 100644 lib/merkle_cache.ex diff --git a/lib/block_process.ex b/lib/block_process.ex index 02be006..0876bfa 100644 --- a/lib/block_process.ex +++ b/lib/block_process.ex @@ -73,7 +73,6 @@ defmodule BlockProcess do def with_block(block_hash, fun) do key = {__MODULE__, :block, block_hash} - # IO.inspect(Profiler.stacktrace()) case Process.get(key, nil) do nil -> block = diff --git a/lib/chain/state.ex b/lib/chain/state.ex index a58c3c7..fd52911 100644 --- a/lib/chain/state.ex +++ b/lib/chain/state.ex @@ -146,7 +146,7 @@ defmodule Chain.State do end end) - state_diff = MerkleTree.difference(Account.tree(acc_a), Account.tree(acc_b)) + state_diff = MerkleCache.difference(Account.tree(acc_a), Account.tree(acc_b)) if map_size(state_diff) > 0 do Map.merge(report, %{ diff --git a/lib/diode.ex b/lib/diode.ex index 19652bb..7f082ad 100644 --- a/lib/diode.ex +++ b/lib/diode.ex @@ -63,6 +63,7 @@ defmodule Diode do base_children = [ worker(Stats, []), + worker(MerkleCache, []), supervisor(Model.Sql), supervisor(Channels), worker(Chain.BlockCache, [ets_extra]), diff --git a/lib/mapmerkletree.ex b/lib/mapmerkletree.ex index a512476..ac2fddf 100644 --- a/lib/mapmerkletree.ex +++ b/lib/mapmerkletree.ex @@ -21,11 +21,11 @@ defmodule MapMerkleTree do end def merkle(tree) do - MerkleTree.copy(tree, MerkleTree2) + MerkleCache.merkle(tree) end def root_hash(tree) do - MerkleTree.root_hash(merkle(tree)) + MerkleCache.root_hash(tree) end def root_hashes(tree) do diff --git a/lib/merkle_cache.ex b/lib/merkle_cache.ex new file mode 100644 index 0000000..ed80013 --- /dev/null +++ b/lib/merkle_cache.ex @@ -0,0 +1,151 @@ +defmodule MerkleCache do + use GenServer + defstruct [:cache] + + def start_link() do + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end + + @impl true + def init(_opts) do + {:ok, %MerkleCache{cache: []}} + end + + def merkle(tree = {MapMerkleTree, _opts, dict}) do + if map_size(dict) > 1000 do + GenServer.call(__MODULE__, {:merkle, dict}) + else + MerkleTree.copy(tree, MerkleTree2) + end + end + + def root_hash(tree = {MapMerkleTree, _opts, dict}) do + if map_size(dict) > 1000 do + GenServer.call(__MODULE__, {:root_hash, dict}) + else + merkle(tree) |> MerkleTree.root_hash() + end + end + + def difference(tree1 = {MapMerkleTree, _opts, map1}, tree2 = {MapMerkleTree, _opts2, map2}) do + if MerkleTree.size(tree1) > 1000 do + GenServer.call(__MODULE__, {:difference, map1, map2}) + else + MerkleTree.difference(tree1, tree2) + end + end + + def difference(tree1 = {MapMerkleTree, _opts, map1}, tree2) do + if MerkleTree.size(tree1) > 1000 do + map2 = MerkleTree.to_list(tree2) |> Map.new() + GenServer.call(__MODULE__, {:difference, map1, map2}) + else + MerkleTree.difference(tree1, tree2) + end + end + + @impl true + def handle_call({:difference, map1, map2}, from, state) do + {:reply, tree1, state} = handle_call({:merkle, map1}, from, state) + root_hash = MerkleTree.root_hash(tree1) + + {state, index} = + Enum.find_index(state.cache, fn %{root_hash: tree_root_hash} -> + tree_root_hash == root_hash + end) + |> case do + nil -> + {do_insert_new(state, MerkleTree.to_list(tree1)), 0} + + index -> + {state, index} + end + + {diff, state} = do_diff(state, map2, index) + {:reply, diff, state} + end + + def handle_call({:root_hash, map}, from, state) do + {:reply, tree, state} = handle_call({:merkle, map}, from, state) + {:reply, MerkleTree.root_hash(tree), state} + end + + def handle_call({:merkle, map}, _from, state = %{cache: cache}) do + tenPercent = div(map_size(map), 10) + lower = max(0, map_size(map) - tenPercent) + upper = map_size(map) + tenPercent + + Enum.find_index(cache, fn %{tree: tree, sample: sample} -> + if lower < MerkleTree.size(tree) and MerkleTree.size(tree) < upper do + score = + Enum.reduce(sample, 0, fn key, score -> + if MerkleTree.get(tree, key) == Map.get(map, key) do + score + 1 + else + score + end + end) + + # IO.puts("score: #{score}") + score * 2 > length(sample) + end + end) + |> case do + nil -> + state = %{cache: [%{tree: tree} | _]} = do_insert_new(state, Map.to_list(map)) + {:reply, tree, state} + + index -> + {diff, state} = do_diff(state, map, index) + %{tree: tree} = Enum.at(cache, index) + + tree2 = + Enum.reduce(diff, tree, fn {key, {_a, b}}, tree -> + if b == nil do + MerkleTree.delete(tree, key) + else + MerkleTree.insert(tree, key, b) + end + end) + + {:reply, tree2, state} + end + end + + defp do_insert_new(state = %{cache: cache}, items) when is_list(items) do + tree = MerkleTree2.new() |> MerkleTree.insert_items(items) + sample = Enum.shuffle(items) |> Enum.take(100) |> Enum.map(&elem(&1, 0)) + selfdiff = %{} + + new = %{ + tree: tree, + sample: sample, + diffs: %{:erlang.phash2(selfdiff) => selfdiff}, + root_hash: MerkleTree.root_hash(tree) + } + + %{state | cache: [new | Enum.take(cache, 10)]} + end + + defp do_diff(state = %{cache: cache}, map, index) do + {_time, {_key, diff, state}} = + :timer.tc(fn -> + %{tree: tree, diffs: diffs} = Enum.at(cache, index) + key = :erlang.phash2(map) + + diff = + Map.get(diffs, key) || + MerkleTree.difference(tree, MapMerkleTree.from_map(map)) + + cache = + List.update_at(cache, index, fn item -> + %{item | diffs: Map.put(diffs, key, diff)} + end) + + {key, diff, %{state | cache: cache}} + end) + + # IO.puts("MerkleCache.do_diff: #{div(time, 1000)}ms = #{key} / #{map_size(diff)}") + {diff, state} + end +end diff --git a/lib/merkletree.ex b/lib/merkletree.ex index 58cc4ad..e9f554e 100644 --- a/lib/merkletree.ex +++ b/lib/merkletree.ex @@ -23,6 +23,10 @@ defmodule MerkleTree do copy(merkle, mod) end + def copy(merkle = {mod, _, _}, mod) do + merkle + end + def copy(merkle, mod) do insert_items(mod.new(), to_list(merkle)) end diff --git a/lib/remote_chain/edge.ex b/lib/remote_chain/edge.ex index c2ae1e9..3db370f 100644 --- a/lib/remote_chain/edge.ex +++ b/lib/remote_chain/edge.ex @@ -152,11 +152,6 @@ defmodule RemoteChain.Edge do end) end - # Moonbeam.estimate_gas(Base16.encode(CallPermit.address()), Base16.encode(call)) - # |> IO.inspect(label: "estimate_gas") - # {:error, %{"message" => error}} -> - # error(error) - tx = Shell.raw(CallPermit.wallet(), call, to: CallPermit.address(), diff --git a/notes.exs b/notes.exs index 2ddf8b0..0170b99 100644 --- a/notes.exs +++ b/notes.exs @@ -1,3 +1,54 @@ +# 10th May 2024 + +bns = 0xaf60faa5cd840b724742f1af116168276112d6a6 +{acc, nr} = Chain.with_peak(fn block -> {Chain.Block.state(block) |> Chain.State.ensure_account( +bns), Chain.Block.number(block)} end) +File.write!("bns_account_#{nr}.etf", :erlang.term_to_binary(acc)) + +Logger.configure(level: :warning) + +accs = ~w(bns_account_7144000.etf bns_account_7144224.etf bns_account_7144861.etf + bns_account_7145072.etf bns_account_7145706.etf) |> Enum.map(&File.read!(&1) |> :erlang.binary_to_term() |> Map.put(:root_hash, nil)) + +check = fn acc -> + {time1, root1} = :timer.tc(fn -> MerkleTree.copy(Chain.Account.tree(acc), MerkleTree2) |> MerkleTree.root_hash() end) + {time2, root2} = :timer.tc(fn -> Chain.Account.root_hash(acc) end) + ^root1 = root2 + {div(time1, 1000), div(time2, 1000), time1 / time2} +end + +check2 = fn acc -> + {time1, root1} = :timer.tc(fn -> MerkleTree.copy(Chain.Account.tree(acc), MerkleTree2) |> MerkleTree.root_hash() end) + {time2, root2} = :timer.tc(fn -> MerkleCache.root_hash(Chain.Account.tree(acc)) end) + ^root1 = root2 + {div(time1, 1000), div(time2, 1000), time1 / time2} +end + +check_diff = fn acc, acc2 -> + {time1, diff1} = :timer.tc(fn -> MerkleTree.difference(Chain.Account.tree(acc), Chain.Account.tree(acc2)) end) + {time2, diff2} = :timer.tc(fn -> MerkleCache.difference(Chain.Account.tree(acc), Chain.Account.tree(acc2)) end) + ^diff1 = diff2 + {div(time1, 1000), div(time2, 1000), time1 / time2} +end + +:timer.tc(fn -> Chain.Account.root_hash(hd(accs)) end) +:timer.tc(fn -> Chain.Account.root_hash(Enum.at(accs, 1)) end) + +check.(Enum.at(accs, 1)) +check2.(Enum.at(accs, 1)) +check_diff.(Enum.at(accs, 1), Enum.at(accs, 2)) +check_diff.(Enum.at(accs, 1), Enum.at(accs, 1)) +check_diff.(Enum.at(accs, 2), Enum.at(accs, 1)) + +loop = fn loop, acc -> + Chain.Account.root_hash(acc) + loop.(loop, acc) +end + +pid = spawn(fn -> loop.(loop, Enum.at(accs, 1)) end) +spawn(fn -> Profiler.fprof(MerkleCache) end) + + # 9th May 2024 node = :global.whereis_name({RemoteChain.NodeProxy, Chains.Moonbeam})