From a9cbfd785983357325aaeae3336fd1efc5798d69 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Mon, 14 Nov 2022 10:13:47 +0100 Subject: [PATCH 01/19] Add functions to read/write inputs from/to disk --- lib/archethic/db/embedded_impl/inputs.ex | 86 +++++++++++++++++++ .../db/embedded_impl/inputs_test.exs | 65 ++++++++++++++ .../transaction_input_test.exs | 28 ++++++ 3 files changed, 179 insertions(+) create mode 100644 lib/archethic/db/embedded_impl/inputs.ex create mode 100644 test/archethic/db/embedded_impl/inputs_test.exs create mode 100644 test/archethic/transaction_chain/transaction_input_test.exs diff --git a/lib/archethic/db/embedded_impl/inputs.ex b/lib/archethic/db/embedded_impl/inputs.ex new file mode 100644 index 000000000..aaf3d48c9 --- /dev/null +++ b/lib/archethic/db/embedded_impl/inputs.ex @@ -0,0 +1,86 @@ +defmodule Archethic.DB.EmbeddedImpl.Inputs do + @moduledoc """ + Inputs are stored by destination address. 1 file per address + There will be many files + """ + + use GenServer + + alias Archethic.TransactionChain.VersionedTransactionInput + alias Archethic.Utils + + def start_link(opts \\ []) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + @spec append_inputs(inputs :: list(VersionedTransactionInput.t()), address :: binary()) :: :ok + def append_inputs(inputs, address) do + GenServer.call(__MODULE__, {:append_inputs, inputs, address}) + end + + @spec get_inputs(address :: binary()) :: list(VersionedTransactionInput.t()) + def get_inputs(address) do + GenServer.call(__MODULE__, {:get_inputs, address}) + end + + def init(opts) do + db_path = Keyword.get(opts, :path) + input_path = Path.join(db_path, "inputs") + + # setup folder + :ok = File.mkdir_p!(input_path) + + {:ok, %{input_path: input_path}} + end + + def handle_call({:get_inputs, address}, _, state = %{input_path: input_path}) do + inputs = + address + |> address_to_filename(input_path) + |> read_inputs_from_file() + + {:reply, inputs, state} + end + + def handle_call({:append_inputs, inputs, address}, _, state = %{input_path: input_path}) do + address + |> address_to_filename(input_path) + |> write_inputs_to_file(inputs) + + {:reply, :ok, state} + end + + defp read_inputs_from_file(filename) do + case File.read(filename) do + {:ok, bin} -> + deserialize_inputs_file(bin, []) + + _ -> + [] + end + end + + defp write_inputs_to_file(filename, inputs) do + inputs_bin = + inputs + |> Enum.map(&VersionedTransactionInput.serialize(&1)) + |> :erlang.list_to_bitstring() + |> Utils.wrap_binary() + + File.write!(filename, inputs_bin, [:append, :binary]) + end + + defp deserialize_inputs_file(bin, acc) do + if bit_size(bin) < 8 do + # less than a byte, we are in the padding of wrap_binary + acc + else + # todo: ORDER? + # deserialize ONE input and return the rest + {input, rest} = VersionedTransactionInput.deserialize(bin) + deserialize_inputs_file(rest, [input | acc]) + end + end + + defp address_to_filename(address, path), do: Path.join([path, Base.encode16(address)]) +end diff --git a/test/archethic/db/embedded_impl/inputs_test.exs b/test/archethic/db/embedded_impl/inputs_test.exs new file mode 100644 index 000000000..047b70e35 --- /dev/null +++ b/test/archethic/db/embedded_impl/inputs_test.exs @@ -0,0 +1,65 @@ +defmodule Archethic.DB.EmbeddedImpl.InputsTest do + use ArchethicCase + + alias Archethic.DB.EmbeddedImpl.Inputs + + alias Archethic.TransactionChain.VersionedTransactionInput + alias Archethic.TransactionChain.TransactionInput + + setup do + db_path = Application.app_dir(:archethic, "data_test") + File.mkdir_p!(db_path) + + {:ok, _} = Inputs.start_link(path: db_path) + + on_exit(fn -> + File.rm_rf!(db_path) + end) + + %{db_path: db_path} + end + + describe "Append/Get" do + test "returns empty when there is none", %{db_path: db_path} do + address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + + assert [] = Inputs.get_inputs(address) + end + + test "returns the inputs that were appended", %{db_path: db_path} do + address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + address2 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + address3 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + + inputs = [ + %VersionedTransactionInput{ + protocol_version: 1, + input: %TransactionInput{ + amount: 1, + type: :UCO, + from: address2, + reward?: true, + spent?: true, + timestamp: DateTime.utc_now() + } + }, + %VersionedTransactionInput{ + protocol_version: 1, + input: %TransactionInput{ + amount: 2, + type: :UCO, + from: address3, + reward?: true, + spent?: true, + timestamp: DateTime.utc_now() + } + } + ] + + Inputs.append_inputs(inputs, address) + assert inputs = Inputs.get_inputs(address) + assert [] = Inputs.get_inputs(address2) + assert [] = Inputs.get_inputs(address3) + end + end +end diff --git a/test/archethic/transaction_chain/transaction_input_test.exs b/test/archethic/transaction_chain/transaction_input_test.exs new file mode 100644 index 000000000..3f02c3305 --- /dev/null +++ b/test/archethic/transaction_chain/transaction_input_test.exs @@ -0,0 +1,28 @@ +defmodule Archethic.TransactionChain.TransactionInputTest do + use ExUnit.Case + + alias Archethic.Mining + + alias Archethic.TransactionChain.TransactionInput + + describe "serialization/deserialization workflow" do + test "should return the same transaction after serialization and deserialization" do + input = %TransactionInput{ + amount: 1, + type: :UCO, + from: <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>>, + reward?: true, + spent?: true, + timestamp: DateTime.utc_now() + } + + protocol_version = Mining.protocol_version() + + assert input = + TransactionInput.deserialize( + TransactionInput.serialize(input, protocol_version), + protocol_version + ) + end + end +end From b7cc03e7dae290742ce165c4af1fa41fe152d179 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Mon, 14 Nov 2022 14:59:20 +0100 Subject: [PATCH 02/19] Move spent UCO from ETS to disk there is one file per address --- .../account/mem_tables/uco_ledger.ex | 83 +++++++--- lib/archethic/db/embedded_impl/inputs.ex | 46 ++---- .../account/mem_tables/uco_ledger_test.exs | 149 +++++++++++------- .../db/embedded_impl/inputs_test.exs | 12 +- 4 files changed, 166 insertions(+), 124 deletions(-) diff --git a/lib/archethic/account/mem_tables/uco_ledger.ex b/lib/archethic/account/mem_tables/uco_ledger.ex index 489c4496d..adae40678 100644 --- a/lib/archethic/account/mem_tables/uco_ledger.ex +++ b/lib/archethic/account/mem_tables/uco_ledger.ex @@ -4,6 +4,8 @@ defmodule Archethic.Account.MemTables.UCOLedger do @ledger_table :archethic_uco_ledger @unspent_output_index_table :archethic_uco_unspent_output_index + alias Archethic.DB.EmbeddedImpl.Inputs + alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations.UnspentOutput alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations.VersionedUnspentOutput @@ -120,39 +122,74 @@ defmodule Archethic.Account.MemTables.UCOLedger do end @doc """ - Spend all the unspent outputs for the given address + Spend all the unspent outputs for the given address. + Spent UTXO are removed from the ETS table """ @spec spend_all_unspent_outputs(binary()) :: :ok def spend_all_unspent_outputs(address) do - @unspent_output_index_table - |> :ets.lookup(address) - |> Enum.each(&:ets.update_element(@ledger_table, &1, {3, true})) + inputs = + @unspent_output_index_table + |> :ets.lookup(address) + |> Enum.map(fn {to, from} -> + [{_, amount, _, timestamp, reward?, protocol_version}] = + :ets.lookup(@ledger_table, {to, from}) + + %VersionedTransactionInput{ + protocol_version: protocol_version, + input: %TransactionInput{ + from: from, + amount: amount, + spent?: true, + reward?: reward?, + timestamp: timestamp, + type: :UCO + } + } + end) - :ok + Inputs.append_inputs(inputs, address) + + Enum.each(inputs, fn %VersionedTransactionInput{input: %TransactionInput{from: from}} -> + :ets.delete(@ledger_table, {address, from}) + end) end @doc """ Retrieve the entire inputs for a given address (spent or unspent) + Unspent come from the ETS table + Spent come from local disk """ @spec get_inputs(binary()) :: list(VersionedTransactionInput.t()) def get_inputs(address) when is_binary(address) do - @unspent_output_index_table - |> :ets.lookup(address) - |> Enum.map(fn {_, from} -> - [{_, amount, spent?, timestamp, reward?, protocol_version}] = - :ets.lookup(@ledger_table, {address, from}) - - %VersionedTransactionInput{ - input: %TransactionInput{ - from: from, - amount: amount, - spent?: spent?, - type: :UCO, - timestamp: timestamp, - reward?: reward? - }, - protocol_version: protocol_version - } - end) + unspent = + @unspent_output_index_table + |> :ets.lookup(address) + |> Enum.reduce([], fn {_, from}, acc -> + case :ets.lookup(@ledger_table, {address, from}) do + [] -> + # the output has been spent and is now written on disk + acc + + [{_, amount, spent?, timestamp, reward?, protocol_version}] -> + [ + %VersionedTransactionInput{ + input: %TransactionInput{ + from: from, + amount: amount, + spent?: spent?, + type: :UCO, + timestamp: timestamp, + reward?: reward? + }, + protocol_version: protocol_version + } + | acc + ] + end + end) + |> Enum.reverse() + + spent = Inputs.get_inputs(address) + Enum.concat(spent, unspent) end end diff --git a/lib/archethic/db/embedded_impl/inputs.ex b/lib/archethic/db/embedded_impl/inputs.ex index aaf3d48c9..e3609ec81 100644 --- a/lib/archethic/db/embedded_impl/inputs.ex +++ b/lib/archethic/db/embedded_impl/inputs.ex @@ -4,56 +4,28 @@ defmodule Archethic.DB.EmbeddedImpl.Inputs do There will be many files """ - use GenServer - + alias Archethic.DB.EmbeddedImpl alias Archethic.TransactionChain.VersionedTransactionInput alias Archethic.Utils - def start_link(opts \\ []) do - GenServer.start_link(__MODULE__, opts, name: __MODULE__) - end - @spec append_inputs(inputs :: list(VersionedTransactionInput.t()), address :: binary()) :: :ok def append_inputs(inputs, address) do - GenServer.call(__MODULE__, {:append_inputs, inputs, address}) + filename = address_to_filename(address) + :ok = File.mkdir_p!(Path.dirname(filename)) + write_inputs_to_file(filename, inputs) end @spec get_inputs(address :: binary()) :: list(VersionedTransactionInput.t()) def get_inputs(address) do - GenServer.call(__MODULE__, {:get_inputs, address}) - end - - def init(opts) do - db_path = Keyword.get(opts, :path) - input_path = Path.join(db_path, "inputs") - - # setup folder - :ok = File.mkdir_p!(input_path) - - {:ok, %{input_path: input_path}} - end - - def handle_call({:get_inputs, address}, _, state = %{input_path: input_path}) do - inputs = - address - |> address_to_filename(input_path) - |> read_inputs_from_file() - - {:reply, inputs, state} - end - - def handle_call({:append_inputs, inputs, address}, _, state = %{input_path: input_path}) do address - |> address_to_filename(input_path) - |> write_inputs_to_file(inputs) - - {:reply, :ok, state} + |> address_to_filename() + |> read_inputs_from_file() end defp read_inputs_from_file(filename) do case File.read(filename) do {:ok, bin} -> - deserialize_inputs_file(bin, []) + Enum.reverse(deserialize_inputs_file(bin, [])) _ -> [] @@ -70,6 +42,7 @@ defmodule Archethic.DB.EmbeddedImpl.Inputs do File.write!(filename, inputs_bin, [:append, :binary]) end + # instead of ignoring padding, we should count the iteration and stop once over defp deserialize_inputs_file(bin, acc) do if bit_size(bin) < 8 do # less than a byte, we are in the padding of wrap_binary @@ -82,5 +55,6 @@ defmodule Archethic.DB.EmbeddedImpl.Inputs do end end - defp address_to_filename(address, path), do: Path.join([path, Base.encode16(address)]) + defp address_to_filename(address), + do: Path.join([EmbeddedImpl.db_path(), "inputs", Base.encode16(address)]) end diff --git a/test/archethic/account/mem_tables/uco_ledger_test.exs b/test/archethic/account/mem_tables/uco_ledger_test.exs index d75c114bd..5c3d9658c 100644 --- a/test/archethic/account/mem_tables/uco_ledger_test.exs +++ b/test/archethic/account/mem_tables/uco_ledger_test.exs @@ -13,15 +13,19 @@ defmodule Archethic.Account.MemTables.UCOLedgerTest do test "add_unspent_output/3 should insert a new entry in the tables" do {:ok, _pid} = UCOLedger.start_link() + alice2 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + bob3 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + charlie10 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + :ok = UCOLedger.add_unspent_output( - "@Alice2", + alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Bob3", + from: bob3, amount: 300_000_000, type: :UCO, - timestamp: ~U[2022-10-11 09:24:01.879Z] + timestamp: ~U[2022-10-11 09:24:01Z] }, protocol_version: 1 } @@ -29,48 +33,65 @@ defmodule Archethic.Account.MemTables.UCOLedgerTest do :ok = UCOLedger.add_unspent_output( - "@Alice2", + alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Charlie10", + from: charlie10, amount: 100_000_000, type: :UCO, - timestamp: ~U[2022-10-11 09:24:01.879Z] + timestamp: ~U[2022-10-11 09:24:01Z] }, protocol_version: 1 } ) - assert [ - {{"@Alice2", "@Bob3"}, 300_000_000, false, ~U[2022-10-11 09:24:01.879Z], false, 1}, - {{"@Alice2", "@Charlie10"}, 100_000_000, false, ~U[2022-10-11 09:24:01.879Z], false, - 1} - ] = :ets.tab2list(:archethic_uco_ledger) + # cannot rely on ordering because of randomness + ledger_content = :ets.tab2list(:archethic_uco_ledger) + assert length(ledger_content) == 2 + + assert Enum.any?( + ledger_content, + &(&1 == + {{alice2, bob3}, 300_000_000, false, ~U[2022-10-11 09:24:01Z], false, 1}) + ) + + assert Enum.any?( + ledger_content, + &(&1 == + {{alice2, charlie10}, 100_000_000, false, ~U[2022-10-11 09:24:01Z], false, 1}) + ) - assert [ - {"@Alice2", "@Bob3"}, - {"@Alice2", "@Charlie10"} - ] = :ets.tab2list(:archethic_uco_unspent_output_index) + index_content = :ets.tab2list(:archethic_uco_unspent_output_index) + assert length(index_content) == 2 + assert Enum.any?(index_content, &(&1 == {alice2, bob3})) + assert Enum.any?(index_content, &(&1 == {alice2, charlie10})) end describe "get_unspent_outputs/1" do test "should return an empty list" do {:ok, _pid} = UCOLedger.start_link() - assert [] = UCOLedger.get_unspent_outputs("@Alice2") + + alice2 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + + assert [] = UCOLedger.get_unspent_outputs(alice2) end test "should return unspent transaction outputs" do {:ok, _pid} = UCOLedger.start_link() + alice2 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + bob3 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + charlie10 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + :ok = UCOLedger.add_unspent_output( - "@Alice2", + alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Bob3", + from: bob3, amount: 300_000_000, type: :UCO, - timestamp: ~U[2022-10-11 09:24:01.879Z] + timestamp: ~U[2022-10-11 09:24:01Z] }, protocol_version: 1 } @@ -78,13 +99,13 @@ defmodule Archethic.Account.MemTables.UCOLedgerTest do :ok = UCOLedger.add_unspent_output( - "@Alice2", + alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Charlie10", + from: charlie10, amount: 100_000_000, type: :UCO, - timestamp: ~U[2022-10-10 09:27:17.846Z] + timestamp: ~U[2022-10-10 09:27:17Z] }, protocol_version: 1 } @@ -93,38 +114,42 @@ defmodule Archethic.Account.MemTables.UCOLedgerTest do assert [ %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Charlie10", + from: ^charlie10, amount: 100_000_000, type: :UCO, - timestamp: ~U[2022-10-10 09:27:17.846Z] + timestamp: ~U[2022-10-10 09:27:17Z] }, protocol_version: 1 }, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Bob3", + from: ^bob3, amount: 300_000_000, type: :UCO, - timestamp: ~U[2022-10-11 09:24:01.879Z] + timestamp: ~U[2022-10-11 09:24:01Z] }, protocol_version: 1 } - ] = UCOLedger.get_unspent_outputs("@Alice2") + ] = UCOLedger.get_unspent_outputs(alice2) end end test "spend_all_unspent_outputs/1 should mark all entries for an address as spent" do {:ok, _pid} = UCOLedger.start_link() + alice2 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + bob3 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + charlie10 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + :ok = UCOLedger.add_unspent_output( - "@Alice2", + alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Bob3", + from: bob3, amount: 300_000_000, type: :UCO, - timestamp: ~U[2022-10-11 09:24:01.879Z] + timestamp: ~U[2022-10-11 09:24:01Z] }, protocol_version: 1 } @@ -132,35 +157,39 @@ defmodule Archethic.Account.MemTables.UCOLedgerTest do :ok = UCOLedger.add_unspent_output( - "@Alice2", + alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Charlie10", + from: charlie10, amount: 100_000_000, type: :UCO, - timestamp: ~U[2022-10-11 09:24:01.879Z] + timestamp: ~U[2022-10-11 09:24:01Z] }, protocol_version: 1 } ) - :ok = UCOLedger.spend_all_unspent_outputs("@Alice2") - assert [] = UCOLedger.get_unspent_outputs("@Alice2") + :ok = UCOLedger.spend_all_unspent_outputs(alice2) + assert [] = UCOLedger.get_unspent_outputs(alice2) end describe "get_inputs/1" do test "convert unspent outputs" do {:ok, _pid} = UCOLedger.start_link() + alice2 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + bob3 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + charlie10 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + :ok = UCOLedger.add_unspent_output( - "@Alice2", + alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Bob3", + from: bob3, amount: 300_000_000, type: :UCO, - timestamp: ~U[2022-10-11 09:24:01.879Z] + timestamp: ~U[2022-10-11 09:24:01Z] }, protocol_version: 1 } @@ -168,13 +197,13 @@ defmodule Archethic.Account.MemTables.UCOLedgerTest do :ok = UCOLedger.add_unspent_output( - "@Alice2", + alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Charlie10", + from: charlie10, amount: 100_000_000, type: :UCO, - timestamp: ~U[2022-10-11 09:24:01.879Z] + timestamp: ~U[2022-10-11 09:24:01Z] }, protocol_version: 1 } @@ -183,39 +212,43 @@ defmodule Archethic.Account.MemTables.UCOLedgerTest do assert [ %VersionedTransactionInput{ input: %TransactionInput{ - from: "@Bob3", + from: ^bob3, amount: 300_000_000, spent?: false, type: :UCO, - timestamp: ~U[2022-10-11 09:24:01.879Z] + timestamp: ~U[2022-10-11 09:24:01Z] }, protocol_version: 1 }, %VersionedTransactionInput{ input: %TransactionInput{ - from: "@Charlie10", + from: ^charlie10, amount: 100_000_000, spent?: false, type: :UCO, - timestamp: ~U[2022-10-11 09:24:01.879Z] + timestamp: ~U[2022-10-11 09:24:01Z] }, protocol_version: 1 } - ] = UCOLedger.get_inputs("@Alice2") + ] = UCOLedger.get_inputs(alice2) end test "should convert spent outputs" do {:ok, _pid} = UCOLedger.start_link() + alice2 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + bob3 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + charlie10 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + :ok = UCOLedger.add_unspent_output( - "@Alice2", + alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Bob3", + from: bob3, amount: 300_000_000, type: :UCO, - timestamp: ~U[2022-10-11 09:24:01.879Z] + timestamp: ~U[2022-10-11 09:24:01Z] }, protocol_version: 1 } @@ -223,42 +256,42 @@ defmodule Archethic.Account.MemTables.UCOLedgerTest do :ok = UCOLedger.add_unspent_output( - "@Alice2", + alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Charlie10", + from: charlie10, amount: 100_000_000, type: :UCO, - timestamp: ~U[2022-10-11 09:24:01.879Z] + timestamp: ~U[2022-10-11 09:24:01Z] }, protocol_version: 1 } ) - :ok = UCOLedger.spend_all_unspent_outputs("@Alice2") + :ok = UCOLedger.spend_all_unspent_outputs(alice2) assert [ %VersionedTransactionInput{ input: %TransactionInput{ - from: "@Bob3", + from: ^bob3, amount: 300_000_000, spent?: true, type: :UCO, - timestamp: ~U[2022-10-11 09:24:01.879Z] + timestamp: ~U[2022-10-11 09:24:01Z] }, protocol_version: 1 }, %VersionedTransactionInput{ input: %TransactionInput{ - from: "@Charlie10", + from: ^charlie10, amount: 100_000_000, spent?: true, type: :UCO, - timestamp: ~U[2022-10-11 09:24:01.879Z] + timestamp: ~U[2022-10-11 09:24:01Z] }, protocol_version: 1 } - ] = UCOLedger.get_inputs("@Alice2") + ] = UCOLedger.get_inputs(alice2) end end end diff --git a/test/archethic/db/embedded_impl/inputs_test.exs b/test/archethic/db/embedded_impl/inputs_test.exs index 047b70e35..cf1b4197a 100644 --- a/test/archethic/db/embedded_impl/inputs_test.exs +++ b/test/archethic/db/embedded_impl/inputs_test.exs @@ -10,8 +10,6 @@ defmodule Archethic.DB.EmbeddedImpl.InputsTest do db_path = Application.app_dir(:archethic, "data_test") File.mkdir_p!(db_path) - {:ok, _} = Inputs.start_link(path: db_path) - on_exit(fn -> File.rm_rf!(db_path) end) @@ -20,13 +18,13 @@ defmodule Archethic.DB.EmbeddedImpl.InputsTest do end describe "Append/Get" do - test "returns empty when there is none", %{db_path: db_path} do + test "returns empty when there is none" do address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> assert [] = Inputs.get_inputs(address) end - test "returns the inputs that were appended", %{db_path: db_path} do + test "returns the inputs that were appended" do address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> address2 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> address3 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> @@ -40,7 +38,7 @@ defmodule Archethic.DB.EmbeddedImpl.InputsTest do from: address2, reward?: true, spent?: true, - timestamp: DateTime.utc_now() + timestamp: ~U[2022-11-14 14:54:12Z] } }, %VersionedTransactionInput{ @@ -51,13 +49,13 @@ defmodule Archethic.DB.EmbeddedImpl.InputsTest do from: address3, reward?: true, spent?: true, - timestamp: DateTime.utc_now() + timestamp: ~U[2022-11-14 14:54:12Z] } } ] Inputs.append_inputs(inputs, address) - assert inputs = Inputs.get_inputs(address) + assert ^inputs = Inputs.get_inputs(address) assert [] = Inputs.get_inputs(address2) assert [] = Inputs.get_inputs(address3) end From 3d9fba99303c9a22a4fe1ad9d3cf553b5fa3afbb Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Mon, 14 Nov 2022 16:09:25 +0100 Subject: [PATCH 03/19] Move spent token from ETS to disk there is one file per address --- .../account/mem_tables/token_ledger.ex | 84 +++++--- .../account/mem_tables/token_ledger_test.exs | 186 +++++++++++------- 2 files changed, 171 insertions(+), 99 deletions(-) diff --git a/lib/archethic/account/mem_tables/token_ledger.ex b/lib/archethic/account/mem_tables/token_ledger.ex index c08c99ae9..f6efad135 100644 --- a/lib/archethic/account/mem_tables/token_ledger.ex +++ b/lib/archethic/account/mem_tables/token_ledger.ex @@ -4,6 +4,8 @@ defmodule Archethic.Account.MemTables.TokenLedger do @ledger_table :archethic_token_ledger @unspent_output_index_table :archethic_token_unspent_output_index + alias Archethic.DB.EmbeddedImpl.Inputs + alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations.UnspentOutput alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations.VersionedUnspentOutput @@ -17,8 +19,8 @@ defmodule Archethic.Account.MemTables.TokenLedger do @doc """ Initialize the Token ledger tables: - - Main Token ledger as ETS set ({token, to, from, token_id}, amount, spent?, timestamp, protocol_version) - - Token Unspent Output Index as ETS bag (to, {from, token, token_id}) + - Main Token ledger as ETS set ({to, from, token_address, token_id}, amount, spent?, timestamp, protocol_version) + - Token Unspent Output Index as ETS bag (to, from, token_address, token_id) """ @spec start_link(args :: list()) :: GenServer.on_start() def start_link(args \\ []) do @@ -130,10 +132,35 @@ defmodule Archethic.Account.MemTables.TokenLedger do """ @spec spend_all_unspent_outputs(binary()) :: :ok def spend_all_unspent_outputs(address) do - @unspent_output_index_table - |> :ets.lookup(address) - |> Enum.each(fn {_, from, token_address, token_id} -> - :ets.update_element(@ledger_table, {address, from, token_address, token_id}, {3, true}) + inputs = + @unspent_output_index_table + |> :ets.lookup(address) + |> Enum.map(fn {to, from, token_address, token_id} -> + [{_, amount, _, timestamp, protocol_version}] = + :ets.lookup(@ledger_table, {to, from, token_address, token_id}) + + %VersionedTransactionInput{ + protocol_version: protocol_version, + input: %TransactionInput{ + from: from, + amount: amount, + spent?: true, + timestamp: timestamp, + type: {:token, token_address, token_id} + } + } + end) + + Inputs.append_inputs(inputs, address) + + inputs + |> Enum.each(fn %VersionedTransactionInput{ + input: %TransactionInput{ + from: from, + type: {:token, token_address, token_id} + } + } -> + :ets.delete(@ledger_table, {address, from, token_address, token_id}) end) end @@ -142,22 +169,33 @@ defmodule Archethic.Account.MemTables.TokenLedger do """ @spec get_inputs(binary()) :: list(VersionedTransactionInput.t()) def get_inputs(address) when is_binary(address) do - @unspent_output_index_table - |> :ets.lookup(address) - |> Enum.map(fn {_, from, token_address, token_id} -> - [{_, amount, spent?, timestamp, protocol_version}] = - :ets.lookup(@ledger_table, {address, from, token_address, token_id}) - - %VersionedTransactionInput{ - input: %TransactionInput{ - from: from, - amount: amount, - type: {:token, token_address, token_id}, - spent?: spent?, - timestamp: timestamp - }, - protocol_version: protocol_version - } - end) + unspent = + @unspent_output_index_table + |> :ets.lookup(address) + |> Enum.reduce([], fn {_, from, token_address, token_id}, acc -> + case :ets.lookup(@ledger_table, {address, from, token_address, token_id}) do + [] -> + acc + + [{_, amount, spent?, timestamp, protocol_version}] -> + [ + %VersionedTransactionInput{ + input: %TransactionInput{ + from: from, + amount: amount, + type: {:token, token_address, token_id}, + spent?: spent?, + timestamp: timestamp + }, + protocol_version: protocol_version + } + | acc + ] + end + end) + |> Enum.reverse() + + spent = Inputs.get_inputs(address) + Enum.concat(spent, unspent) end end diff --git a/test/archethic/account/mem_tables/token_ledger_test.exs b/test/archethic/account/mem_tables/token_ledger_test.exs index d80f72c66..74699c888 100644 --- a/test/archethic/account/mem_tables/token_ledger_test.exs +++ b/test/archethic/account/mem_tables/token_ledger_test.exs @@ -2,6 +2,8 @@ defmodule Archethic.Account.MemTables.TokenLedgerTest do @moduledoc false use ExUnit.Case + @token1 <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + alias Archethic.Account.MemTables.TokenLedger alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations.UnspentOutput @@ -14,15 +16,19 @@ defmodule Archethic.Account.MemTables.TokenLedgerTest do test "add_unspent_output/3 insert a new entry in the tables" do {:ok, _pid} = TokenLedger.start_link() + alice2 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + bob3 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + charlie10 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + :ok = TokenLedger.add_unspent_output( - "@Alice2", + alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Bob3", + from: bob3, amount: 300_000_000, - type: {:token, "@Token1", 0}, - timestamp: ~U[2022-10-10 09:27:17.846Z] + type: {:token, @token1, 0}, + timestamp: ~U[2022-10-10 09:27:17Z] }, protocol_version: 1 } @@ -30,49 +36,65 @@ defmodule Archethic.Account.MemTables.TokenLedgerTest do :ok = TokenLedger.add_unspent_output( - "@Alice2", + alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Charlie10", + from: charlie10, amount: 100_000_000, - type: {:token, "@Token1", 1}, - timestamp: ~U[2022-10-10 09:27:17.846Z] + type: {:token, @token1, 1}, + timestamp: ~U[2022-10-10 09:27:17Z] }, protocol_version: 1 } ) - assert [ - {{"@Alice2", "@Bob3", "@Token1", 0}, 300_000_000, false, - ~U[2022-10-10 09:27:17.846Z], 1}, - {{"@Alice2", "@Charlie10", "@Token1", 1}, 100_000_000, false, - ~U[2022-10-10 09:27:17.846Z], 1} - ] = :ets.tab2list(:archethic_token_ledger) - - assert [ - {"@Alice2", "@Bob3", "@Token1", 0}, - {"@Alice2", "@Charlie10", "@Token1", 1} - ] = :ets.tab2list(:archethic_token_unspent_output_index) + ledger_content = :ets.tab2list(:archethic_token_ledger) + assert length(ledger_content) == 2 + + assert Enum.any?( + ledger_content, + &(&1 == + {{alice2, bob3, @token1, 0}, 300_000_000, false, ~U[2022-10-10 09:27:17Z], 1}) + ) + + assert Enum.any?( + ledger_content, + &(&1 == + {{alice2, charlie10, @token1, 1}, 100_000_000, false, ~U[2022-10-10 09:27:17Z], + 1}) + ) + + index_content = :ets.tab2list(:archethic_token_unspent_output_index) + assert length(index_content) == 2 + assert Enum.any?(index_content, &(&1 == {alice2, bob3, @token1, 0})) + assert Enum.any?(index_content, &(&1 == {alice2, charlie10, @token1, 1})) end describe "get_unspent_outputs/1" do test "should return an empty list when there are not entries" do {:ok, _pid} = TokenLedger.start_link() - assert [] = TokenLedger.get_unspent_outputs("@Alice2") + + alice2 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + + assert [] = TokenLedger.get_unspent_outputs(alice2) end test "should return unspent transaction outputs" do {:ok, _pid} = TokenLedger.start_link() + alice2 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + bob3 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + charlie10 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + :ok = TokenLedger.add_unspent_output( - "@Alice2", + alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Bob3", + from: bob3, amount: 300_000_000, - type: {:token, "@Token1", 0}, - timestamp: ~U[2022-10-10 09:27:17.846Z] + type: {:token, @token1, 0}, + timestamp: ~U[2022-10-10 09:27:17Z] }, protocol_version: 1 } @@ -80,13 +102,13 @@ defmodule Archethic.Account.MemTables.TokenLedgerTest do :ok = TokenLedger.add_unspent_output( - "@Alice2", + alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Charlie10", + from: charlie10, amount: 100_000_000, - type: {:token, "@Token1", 1}, - timestamp: ~U[2022-10-10 09:27:17.846Z] + type: {:token, @token1, 1}, + timestamp: ~U[2022-10-10 09:27:17Z] }, protocol_version: 1 } @@ -95,69 +117,77 @@ defmodule Archethic.Account.MemTables.TokenLedgerTest do assert [ %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Charlie10", + from: ^charlie10, amount: 100_000_000, - type: {:token, "@Token1", 1}, - timestamp: ~U[2022-10-10 09:27:17.846Z] + type: {:token, @token1, 1}, + timestamp: ~U[2022-10-10 09:27:17Z] }, protocol_version: 1 }, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Bob3", + from: ^bob3, amount: 300_000_000, - type: {:token, "@Token1", 0}, - timestamp: ~U[2022-10-10 09:27:17.846Z] + type: {:token, @token1, 0}, + timestamp: ~U[2022-10-10 09:27:17Z] }, protocol_version: 1 } - ] = TokenLedger.get_unspent_outputs("@Alice2") + ] = TokenLedger.get_unspent_outputs(alice2) end end test "spend_all_unspent_outputs/1 should mark all entries for an address as spent" do {:ok, _pid} = TokenLedger.start_link() + alice2 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + bob3 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + charlie10 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + :ok = - TokenLedger.add_unspent_output("@Alice2", %VersionedUnspentOutput{ + TokenLedger.add_unspent_output(alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Bob3", + from: bob3, amount: 300_000_000, - type: {:token, "@Token1", 0}, - timestamp: ~U[2022-10-10 09:27:17.846Z] + type: {:token, @token1, 0}, + timestamp: ~U[2022-10-10 09:27:17Z] }, protocol_version: 1 }) :ok = - TokenLedger.add_unspent_output("@Alice2", %VersionedUnspentOutput{ + TokenLedger.add_unspent_output(alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Charlie10", + from: charlie10, amount: 100_000_000, - type: {:token, "@Token1", 1}, - timestamp: ~U[2022-10-10 09:27:17.846Z] + type: {:token, @token1, 1}, + timestamp: ~U[2022-10-10 09:27:17Z] }, protocol_version: 1 }) - :ok = TokenLedger.spend_all_unspent_outputs("@Alice2") + :ok = TokenLedger.spend_all_unspent_outputs(alice2) - assert [] = TokenLedger.get_unspent_outputs("@Alice2") + assert [] = TokenLedger.get_unspent_outputs(alice2) end describe "get_inputs/1" do test "convert unspent outputs" do {:ok, _pid} = TokenLedger.start_link() + alice2 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + bob3 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + charlie10 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + :ok = TokenLedger.add_unspent_output( - "@Alice2", + alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Bob3", + from: bob3, amount: 300_000_000, - type: {:token, "@Token1", 0}, - timestamp: ~U[2022-10-10 09:27:17.846Z] + type: {:token, @token1, 0}, + timestamp: ~U[2022-10-10 09:27:17Z] }, protocol_version: 1 } @@ -165,13 +195,13 @@ defmodule Archethic.Account.MemTables.TokenLedgerTest do :ok = TokenLedger.add_unspent_output( - "@Alice2", + alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Charlie10", + from: charlie10, amount: 100_000_000, - type: {:token, "@Token1", 1}, - timestamp: ~U[2022-10-10 09:27:17.846Z] + type: {:token, @token1, 1}, + timestamp: ~U[2022-10-10 09:27:17Z] }, protocol_version: 1 } @@ -180,39 +210,43 @@ defmodule Archethic.Account.MemTables.TokenLedgerTest do assert [ %VersionedTransactionInput{ input: %TransactionInput{ - from: "@Bob3", + from: ^bob3, amount: 300_000_000, - type: {:token, "@Token1", 0}, + type: {:token, @token1, 0}, spent?: false, - timestamp: ~U[2022-10-10 09:27:17.846Z] + timestamp: ~U[2022-10-10 09:27:17Z] }, protocol_version: 1 }, %VersionedTransactionInput{ input: %TransactionInput{ - from: "@Charlie10", + from: ^charlie10, amount: 100_000_000, - type: {:token, "@Token1", 1}, + type: {:token, @token1, 1}, spent?: false, - timestamp: ~U[2022-10-10 09:27:17.846Z] + timestamp: ~U[2022-10-10 09:27:17Z] }, protocol_version: 1 } - ] = TokenLedger.get_inputs("@Alice2") + ] = TokenLedger.get_inputs(alice2) end test "should convert spent outputs" do {:ok, _pid} = TokenLedger.start_link() + alice2 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + bob3 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + charlie10 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + :ok = TokenLedger.add_unspent_output( - "@Alice2", + alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Bob3", + from: bob3, amount: 300_000_000, - type: {:token, "@Token1", 0}, - timestamp: ~U[2022-10-10 09:27:17.846Z] + type: {:token, @token1, 0}, + timestamp: ~U[2022-10-10 09:27:17Z] }, protocol_version: 1 } @@ -220,42 +254,42 @@ defmodule Archethic.Account.MemTables.TokenLedgerTest do :ok = TokenLedger.add_unspent_output( - "@Alice2", + alice2, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Charlie10", + from: charlie10, amount: 100_000_000, - type: {:token, "@Token1", 1}, - timestamp: ~U[2022-10-10 09:27:17.846Z] + type: {:token, @token1, 1}, + timestamp: ~U[2022-10-10 09:27:17Z] }, protocol_version: 1 } ) - :ok = TokenLedger.spend_all_unspent_outputs("@Alice2") + :ok = TokenLedger.spend_all_unspent_outputs(alice2) assert [ %VersionedTransactionInput{ input: %TransactionInput{ - from: "@Bob3", + from: ^bob3, amount: 300_000_000, - type: {:token, "@Token1", 0}, + type: {:token, @token1, 0}, spent?: true, - timestamp: ~U[2022-10-10 09:27:17.846Z] + timestamp: ~U[2022-10-10 09:27:17Z] }, protocol_version: 1 }, %VersionedTransactionInput{ input: %TransactionInput{ - from: "@Charlie10", + from: ^charlie10, amount: 100_000_000, - type: {:token, "@Token1", 1}, + type: {:token, @token1, 1}, spent?: true, - timestamp: ~U[2022-10-10 09:27:17.846Z] + timestamp: ~U[2022-10-10 09:27:17Z] }, protocol_version: 1 } - ] = TokenLedger.get_inputs("@Alice2") + ] = TokenLedger.get_inputs(alice2) end end end From 9930a8038b993e915f6a7cb692386b7c8726f9c8 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Mon, 14 Nov 2022 18:21:15 +0100 Subject: [PATCH 04/19] Prefix the inputs filename with the ledger type we need the prefix to avoid one ledger overriding the content of the other --- .../account/mem_tables/token_ledger.ex | 4 +- .../account/mem_tables/uco_ledger.ex | 4 +- lib/archethic/db/embedded_impl/inputs.ex | 29 ++-- test/archethic/account_test.exs | 133 ++++++++++++++++-- .../db/embedded_impl/inputs_test.exs | 50 ++++++- .../transaction_input_test.exs | 4 +- 6 files changed, 193 insertions(+), 31 deletions(-) diff --git a/lib/archethic/account/mem_tables/token_ledger.ex b/lib/archethic/account/mem_tables/token_ledger.ex index f6efad135..245f0f600 100644 --- a/lib/archethic/account/mem_tables/token_ledger.ex +++ b/lib/archethic/account/mem_tables/token_ledger.ex @@ -151,7 +151,7 @@ defmodule Archethic.Account.MemTables.TokenLedger do } end) - Inputs.append_inputs(inputs, address) + Inputs.append_inputs(:token, inputs, address) inputs |> Enum.each(fn %VersionedTransactionInput{ @@ -195,7 +195,7 @@ defmodule Archethic.Account.MemTables.TokenLedger do end) |> Enum.reverse() - spent = Inputs.get_inputs(address) + spent = Inputs.get_inputs(:token, address) Enum.concat(spent, unspent) end end diff --git a/lib/archethic/account/mem_tables/uco_ledger.ex b/lib/archethic/account/mem_tables/uco_ledger.ex index adae40678..e103769b5 100644 --- a/lib/archethic/account/mem_tables/uco_ledger.ex +++ b/lib/archethic/account/mem_tables/uco_ledger.ex @@ -147,7 +147,7 @@ defmodule Archethic.Account.MemTables.UCOLedger do } end) - Inputs.append_inputs(inputs, address) + Inputs.append_inputs(:UCO, inputs, address) Enum.each(inputs, fn %VersionedTransactionInput{input: %TransactionInput{from: from}} -> :ets.delete(@ledger_table, {address, from}) @@ -189,7 +189,7 @@ defmodule Archethic.Account.MemTables.UCOLedger do end) |> Enum.reverse() - spent = Inputs.get_inputs(address) + spent = Inputs.get_inputs(:UCO, address) Enum.concat(spent, unspent) end end diff --git a/lib/archethic/db/embedded_impl/inputs.ex b/lib/archethic/db/embedded_impl/inputs.ex index e3609ec81..76af44104 100644 --- a/lib/archethic/db/embedded_impl/inputs.ex +++ b/lib/archethic/db/embedded_impl/inputs.ex @@ -8,17 +8,23 @@ defmodule Archethic.DB.EmbeddedImpl.Inputs do alias Archethic.TransactionChain.VersionedTransactionInput alias Archethic.Utils - @spec append_inputs(inputs :: list(VersionedTransactionInput.t()), address :: binary()) :: :ok - def append_inputs(inputs, address) do - filename = address_to_filename(address) + @type ledger :: :token | :UCO + + @spec append_inputs( + ledger :: ledger, + inputs :: list(VersionedTransactionInput.t()), + address :: binary() + ) :: :ok + def append_inputs(ledger, inputs, address) do + filename = address_to_filename(address, ledger) :ok = File.mkdir_p!(Path.dirname(filename)) write_inputs_to_file(filename, inputs) end - @spec get_inputs(address :: binary()) :: list(VersionedTransactionInput.t()) - def get_inputs(address) do + @spec get_inputs(ledger :: ledger, address :: binary()) :: list(VersionedTransactionInput.t()) + def get_inputs(ledger, address) do address - |> address_to_filename() + |> address_to_filename(ledger) |> read_inputs_from_file() end @@ -55,6 +61,13 @@ defmodule Archethic.DB.EmbeddedImpl.Inputs do end end - defp address_to_filename(address), - do: Path.join([EmbeddedImpl.db_path(), "inputs", Base.encode16(address)]) + defp address_to_filename(address, ledger) do + prefix = + case ledger do + :UCO -> "uco" + :token -> "token" + end + + Path.join([EmbeddedImpl.db_path(), "inputs", prefix, Base.encode16(address)]) + end end diff --git a/test/archethic/account_test.exs b/test/archethic/account_test.exs index fe528f062..d8222ce20 100644 --- a/test/archethic/account_test.exs +++ b/test/archethic/account_test.exs @@ -13,6 +13,8 @@ defmodule Archethic.AccountTest do alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations.VersionedUnspentOutput alias Archethic.TransactionChain.Transaction + alias Archethic.TransactionChain.TransactionInput + alias Archethic.TransactionChain.VersionedTransactionInput alias Archethic.Reward.MemTables.RewardTokens alias Archethic.Reward.MemTablesLoader, as: RewardMemTableLoader @@ -126,21 +128,31 @@ defmodule Archethic.AccountTest do end test "should return 0 when all the unspent outputs have been spent" do - timestamp = DateTime.utc_now() |> DateTime.truncate(:millisecond) + timestamp = DateTime.utc_now() |> DateTime.truncate(:second) + + alice2_address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + bob3_address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + charlie2_address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + tom10_address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + charlie_token_address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> UCOLedger.add_unspent_output( - "@Alice2", + alice2_address, %VersionedUnspentOutput{ - unspent_output: %UnspentOutput{from: "@Bob3", amount: 300_000_000, timestamp: timestamp}, + unspent_output: %UnspentOutput{ + from: bob3_address, + amount: 300_000_000, + timestamp: timestamp + }, protocol_version: 1 } ) UCOLedger.add_unspent_output( - "@Alice2", + alice2_address, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Tom10", + from: tom10_address, amount: 100_000_000, timestamp: timestamp }, @@ -149,22 +161,121 @@ defmodule Archethic.AccountTest do ) TokenLedger.add_unspent_output( - "@Alice2", + alice2_address, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ - from: "@Charlie2", + from: charlie2_address, amount: 10_000_000_000, - type: {:token, "@CharlieToken", 0}, + type: {:token, charlie_token_address, 0}, timestamp: timestamp }, protocol_version: 1 } ) - UCOLedger.spend_all_unspent_outputs("@Alice2") - TokenLedger.spend_all_unspent_outputs("@Alice2") + UCOLedger.spend_all_unspent_outputs(alice2_address) + TokenLedger.spend_all_unspent_outputs(alice2_address) - assert %{uco: 0, token: %{}} == Account.get_balance("@Alice2") + assert %{uco: 0, token: %{}} == Account.get_balance(alice2_address) end end + + test "should return both UCO and TOKEN spent" do + timestamp = DateTime.utc_now() |> DateTime.truncate(:second) + + start_supervised!(UCOLedger) + start_supervised!(TokenLedger) + + alice2_address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + bob3_address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + charlie2_address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + tom10_address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + charlie_token_address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + + UCOLedger.add_unspent_output( + alice2_address, + %VersionedUnspentOutput{ + unspent_output: %UnspentOutput{ + from: bob3_address, + amount: 300_000_000, + timestamp: timestamp + }, + protocol_version: 1 + } + ) + + UCOLedger.add_unspent_output( + alice2_address, + %VersionedUnspentOutput{ + unspent_output: %UnspentOutput{ + from: tom10_address, + amount: 100_000_000, + timestamp: timestamp + }, + protocol_version: 1 + } + ) + + TokenLedger.add_unspent_output( + alice2_address, + %VersionedUnspentOutput{ + unspent_output: %UnspentOutput{ + from: charlie2_address, + amount: 10_000_000_000, + type: {:token, charlie_token_address, 0}, + timestamp: timestamp + }, + protocol_version: 1 + } + ) + + UCOLedger.spend_all_unspent_outputs(alice2_address) + TokenLedger.spend_all_unspent_outputs(alice2_address) + + inputs = Account.get_inputs(alice2_address) + + assert 3 == length(inputs) + + assert Enum.any?( + inputs, + &(&1 == %VersionedTransactionInput{ + input: %TransactionInput{ + from: charlie2_address, + amount: 10_000_000_000, + type: {:token, charlie_token_address, 0}, + timestamp: timestamp, + spent?: true + }, + protocol_version: 1 + }) + ) + + assert Enum.any?( + inputs, + &(&1 == %VersionedTransactionInput{ + input: %TransactionInput{ + from: tom10_address, + amount: 100_000_000, + type: :UCO, + timestamp: timestamp, + spent?: true + }, + protocol_version: 1 + }) + ) + + assert Enum.any?( + inputs, + &(&1 == %VersionedTransactionInput{ + input: %TransactionInput{ + from: bob3_address, + amount: 300_000_000, + type: :UCO, + timestamp: timestamp, + spent?: true + }, + protocol_version: 1 + }) + ) + end end diff --git a/test/archethic/db/embedded_impl/inputs_test.exs b/test/archethic/db/embedded_impl/inputs_test.exs index cf1b4197a..a16d085ef 100644 --- a/test/archethic/db/embedded_impl/inputs_test.exs +++ b/test/archethic/db/embedded_impl/inputs_test.exs @@ -21,10 +21,11 @@ defmodule Archethic.DB.EmbeddedImpl.InputsTest do test "returns empty when there is none" do address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> - assert [] = Inputs.get_inputs(address) + assert [] = Inputs.get_inputs(:UCO, address) + assert [] = Inputs.get_inputs(:token, address) end - test "returns the inputs that were appended" do + test "returns the UCO inputs that were appended" do address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> address2 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> address3 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> @@ -54,10 +55,47 @@ defmodule Archethic.DB.EmbeddedImpl.InputsTest do } ] - Inputs.append_inputs(inputs, address) - assert ^inputs = Inputs.get_inputs(address) - assert [] = Inputs.get_inputs(address2) - assert [] = Inputs.get_inputs(address3) + Inputs.append_inputs(:UCO, inputs, address) + assert ^inputs = Inputs.get_inputs(:UCO, address) + assert [] = Inputs.get_inputs(:UCO, address2) + assert [] = Inputs.get_inputs(:UCO, address3) + end + + test "returns the TOKEN inputs that were appended" do + address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + address2 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + address3 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + token_address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + + inputs = [ + %VersionedTransactionInput{ + protocol_version: 1, + input: %TransactionInput{ + amount: 1, + type: {:token, token_address, 0}, + from: address2, + reward?: true, + spent?: true, + timestamp: ~U[2022-11-14 14:54:12Z] + } + }, + %VersionedTransactionInput{ + protocol_version: 1, + input: %TransactionInput{ + amount: 2, + type: {:token, token_address, 0}, + from: address3, + reward?: true, + spent?: true, + timestamp: ~U[2022-11-14 14:54:12Z] + } + } + ] + + Inputs.append_inputs(:token, inputs, address) + assert ^inputs = Inputs.get_inputs(:token, address) + assert [] = Inputs.get_inputs(:token, address2) + assert [] = Inputs.get_inputs(:token, address3) end end end diff --git a/test/archethic/transaction_chain/transaction_input_test.exs b/test/archethic/transaction_chain/transaction_input_test.exs index 3f02c3305..5aa0220f3 100644 --- a/test/archethic/transaction_chain/transaction_input_test.exs +++ b/test/archethic/transaction_chain/transaction_input_test.exs @@ -13,12 +13,12 @@ defmodule Archethic.TransactionChain.TransactionInputTest do from: <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>>, reward?: true, spent?: true, - timestamp: DateTime.utc_now() + timestamp: DateTime.utc_now() |> DateTime.truncate(:second) } protocol_version = Mining.protocol_version() - assert input = + assert {^input, _} = TransactionInput.deserialize( TransactionInput.serialize(input, protocol_version), protocol_version From 7763e71a6a80d7b4acd6234c7ca44188439fbc6c Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Mon, 14 Nov 2022 18:31:39 +0100 Subject: [PATCH 05/19] Improve UTXO ledger documentation --- lib/archethic/account/mem_tables/token_ledger.ex | 4 ++++ lib/archethic/account/mem_tables/uco_ledger.ex | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/archethic/account/mem_tables/token_ledger.ex b/lib/archethic/account/mem_tables/token_ledger.ex index 245f0f600..5c0a23031 100644 --- a/lib/archethic/account/mem_tables/token_ledger.ex +++ b/lib/archethic/account/mem_tables/token_ledger.ex @@ -21,6 +21,10 @@ defmodule Archethic.Account.MemTables.TokenLedger do Initialize the Token ledger tables: - Main Token ledger as ETS set ({to, from, token_address, token_id}, amount, spent?, timestamp, protocol_version) - Token Unspent Output Index as ETS bag (to, from, token_address, token_id) + + The ETS ledger caches the unspent UTXO + The ETS index caches both unspent and spent UTXO + Once a UTXO is spent, it is removed from the ledger and written to disk to reduce memory footprint """ @spec start_link(args :: list()) :: GenServer.on_start() def start_link(args \\ []) do diff --git a/lib/archethic/account/mem_tables/uco_ledger.ex b/lib/archethic/account/mem_tables/uco_ledger.ex index e103769b5..4534352fc 100644 --- a/lib/archethic/account/mem_tables/uco_ledger.ex +++ b/lib/archethic/account/mem_tables/uco_ledger.ex @@ -21,6 +21,10 @@ defmodule Archethic.Account.MemTables.UCOLedger do Initialize the UCO ledger tables: - Main UCO ledger as ETS set ({{to, from}, amount, spent?, timestamp, reward?, protocol_version}) - UCO Unspent Output Index as ETS bag (to, from) + + The ETS ledger caches the unspent UTXO + The ETS index caches both unspent and spent UTXO + Once a UTXO is spent, it is removed from the ledger and written to disk to reduce memory footprint """ @spec start_link(args :: list()) :: GenServer.on_start() def start_link(args \\ []) do @@ -123,7 +127,6 @@ defmodule Archethic.Account.MemTables.UCOLedger do @doc """ Spend all the unspent outputs for the given address. - Spent UTXO are removed from the ETS table """ @spec spend_all_unspent_outputs(binary()) :: :ok def spend_all_unspent_outputs(address) do @@ -156,8 +159,6 @@ defmodule Archethic.Account.MemTables.UCOLedger do @doc """ Retrieve the entire inputs for a given address (spent or unspent) - Unspent come from the ETS table - Spent come from local disk """ @spec get_inputs(binary()) :: list(VersionedTransactionInput.t()) def get_inputs(address) when is_binary(address) do From fbf45262ea9cba168e82d8d0cf01670336b092ea Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Tue, 15 Nov 2022 14:26:55 +0100 Subject: [PATCH 06/19] Refactor Inputs API to be able to call append_input from a loop separation between writer (gen server) and reader (function call) --- .../account/mem_tables/token_ledger.ex | 103 ++++++++-------- .../account/mem_tables/uco_ledger.ex | 106 ++++++++--------- lib/archethic/db/embedded_impl/inputs.ex | 73 ------------ .../db/embedded_impl/inputs_reader.ex | 54 +++++++++ .../db/embedded_impl/inputs_writer.ex | 66 +++++++++++ .../account/mem_tables/token_ledger_test.exs | 110 +++++++++++------- .../account/mem_tables/uco_ledger_test.exs | 110 +++++++++++------- .../db/embedded_impl/inputs_test.exs | 38 +++--- 8 files changed, 364 insertions(+), 296 deletions(-) delete mode 100644 lib/archethic/db/embedded_impl/inputs.ex create mode 100644 lib/archethic/db/embedded_impl/inputs_reader.ex create mode 100644 lib/archethic/db/embedded_impl/inputs_writer.ex diff --git a/lib/archethic/account/mem_tables/token_ledger.ex b/lib/archethic/account/mem_tables/token_ledger.ex index 5c0a23031..4614b7dad 100644 --- a/lib/archethic/account/mem_tables/token_ledger.ex +++ b/lib/archethic/account/mem_tables/token_ledger.ex @@ -4,7 +4,8 @@ defmodule Archethic.Account.MemTables.TokenLedger do @ledger_table :archethic_token_ledger @unspent_output_index_table :archethic_token_unspent_output_index - alias Archethic.DB.EmbeddedImpl.Inputs + alias Archethic.DB.EmbeddedImpl.InputsReader + alias Archethic.DB.EmbeddedImpl.InputsWriter alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations.UnspentOutput @@ -22,9 +23,8 @@ defmodule Archethic.Account.MemTables.TokenLedger do - Main Token ledger as ETS set ({to, from, token_address, token_id}, amount, spent?, timestamp, protocol_version) - Token Unspent Output Index as ETS bag (to, from, token_address, token_id) - The ETS ledger caches the unspent UTXO - The ETS index caches both unspent and spent UTXO - Once a UTXO is spent, it is removed from the ledger and written to disk to reduce memory footprint + The ETS ledger and index caches the unspent UTXO + Once a UTXO is spent, it is removed from the ETS and written to disk to reduce memory footprint """ @spec start_link(args :: list()) :: GenServer.on_start() def start_link(args \\ []) do @@ -136,36 +136,31 @@ defmodule Archethic.Account.MemTables.TokenLedger do """ @spec spend_all_unspent_outputs(binary()) :: :ok def spend_all_unspent_outputs(address) do - inputs = - @unspent_output_index_table - |> :ets.lookup(address) - |> Enum.map(fn {to, from, token_address, token_id} -> - [{_, amount, _, timestamp, protocol_version}] = - :ets.lookup(@ledger_table, {to, from, token_address, token_id}) + {:ok, pid} = InputsWriter.start_link(:token, address) - %VersionedTransactionInput{ - protocol_version: protocol_version, - input: %TransactionInput{ - from: from, - amount: amount, - spent?: true, - timestamp: timestamp, - type: {:token, token_address, token_id} - } + @unspent_output_index_table + |> :ets.lookup(address) + |> Enum.each(fn {to, from, token_address, token_id} -> + [{_, amount, _, timestamp, protocol_version}] = + :ets.lookup(@ledger_table, {to, from, token_address, token_id}) + + InputsWriter.append_input(pid, %VersionedTransactionInput{ + protocol_version: protocol_version, + input: %TransactionInput{ + from: from, + amount: amount, + spent?: true, + timestamp: timestamp, + type: {:token, token_address, token_id} } - end) + }) - Inputs.append_inputs(:token, inputs, address) - - inputs - |> Enum.each(fn %VersionedTransactionInput{ - input: %TransactionInput{ - from: from, - type: {:token, token_address, token_id} - } - } -> :ets.delete(@ledger_table, {address, from, token_address, token_id}) end) + + :ets.delete(@unspent_output_index_table, address) + + InputsWriter.stop(pid) end @doc """ @@ -173,33 +168,27 @@ defmodule Archethic.Account.MemTables.TokenLedger do """ @spec get_inputs(binary()) :: list(VersionedTransactionInput.t()) def get_inputs(address) when is_binary(address) do - unspent = - @unspent_output_index_table - |> :ets.lookup(address) - |> Enum.reduce([], fn {_, from, token_address, token_id}, acc -> - case :ets.lookup(@ledger_table, {address, from, token_address, token_id}) do - [] -> - acc - - [{_, amount, spent?, timestamp, protocol_version}] -> - [ - %VersionedTransactionInput{ - input: %TransactionInput{ - from: from, - amount: amount, - type: {:token, token_address, token_id}, - spent?: spent?, - timestamp: timestamp - }, - protocol_version: protocol_version - } - | acc - ] - end - end) - |> Enum.reverse() - - spent = Inputs.get_inputs(:token, address) - Enum.concat(spent, unspent) + spent = InputsReader.get_inputs(:token, address) + + @unspent_output_index_table + |> :ets.lookup(address) + |> Enum.reduce(spent, fn {_, from, token_address, token_id}, acc -> + [{_, amount, spent?, timestamp, protocol_version}] = + :ets.lookup(@ledger_table, {address, from, token_address, token_id}) + + [ + %VersionedTransactionInput{ + input: %TransactionInput{ + from: from, + amount: amount, + type: {:token, token_address, token_id}, + spent?: spent?, + timestamp: timestamp + }, + protocol_version: protocol_version + } + | acc + ] + end) end end diff --git a/lib/archethic/account/mem_tables/uco_ledger.ex b/lib/archethic/account/mem_tables/uco_ledger.ex index 4534352fc..c1968fd1b 100644 --- a/lib/archethic/account/mem_tables/uco_ledger.ex +++ b/lib/archethic/account/mem_tables/uco_ledger.ex @@ -4,7 +4,8 @@ defmodule Archethic.Account.MemTables.UCOLedger do @ledger_table :archethic_uco_ledger @unspent_output_index_table :archethic_uco_unspent_output_index - alias Archethic.DB.EmbeddedImpl.Inputs + alias Archethic.DB.EmbeddedImpl.InputsReader + alias Archethic.DB.EmbeddedImpl.InputsWriter alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations.UnspentOutput @@ -22,9 +23,8 @@ defmodule Archethic.Account.MemTables.UCOLedger do - Main UCO ledger as ETS set ({{to, from}, amount, spent?, timestamp, reward?, protocol_version}) - UCO Unspent Output Index as ETS bag (to, from) - The ETS ledger caches the unspent UTXO - The ETS index caches both unspent and spent UTXO - Once a UTXO is spent, it is removed from the ledger and written to disk to reduce memory footprint + The ETS ledger and index caches the unspent UTXO + Once a UTXO is spent, it is removed from the ETS and written to disk to reduce memory footprint """ @spec start_link(args :: list()) :: GenServer.on_start() def start_link(args \\ []) do @@ -119,7 +119,7 @@ defmodule Archethic.Account.MemTables.UCOLedger do | acc ] - _ -> + [] -> acc end end) @@ -130,31 +130,32 @@ defmodule Archethic.Account.MemTables.UCOLedger do """ @spec spend_all_unspent_outputs(binary()) :: :ok def spend_all_unspent_outputs(address) do - inputs = - @unspent_output_index_table - |> :ets.lookup(address) - |> Enum.map(fn {to, from} -> - [{_, amount, _, timestamp, reward?, protocol_version}] = - :ets.lookup(@ledger_table, {to, from}) + {:ok, pid} = InputsWriter.start_link(:UCO, address) - %VersionedTransactionInput{ - protocol_version: protocol_version, - input: %TransactionInput{ - from: from, - amount: amount, - spent?: true, - reward?: reward?, - timestamp: timestamp, - type: :UCO - } + @unspent_output_index_table + |> :ets.lookup(address) + |> Enum.each(fn {to, from} -> + [{_, amount, _, timestamp, reward?, protocol_version}] = + :ets.lookup(@ledger_table, {to, from}) + + InputsWriter.append_input(pid, %VersionedTransactionInput{ + protocol_version: protocol_version, + input: %TransactionInput{ + from: from, + amount: amount, + spent?: true, + reward?: reward?, + timestamp: timestamp, + type: :UCO } - end) - - Inputs.append_inputs(:UCO, inputs, address) + }) - Enum.each(inputs, fn %VersionedTransactionInput{input: %TransactionInput{from: from}} -> - :ets.delete(@ledger_table, {address, from}) + :ets.delete(@ledger_table, {to, from}) end) + + :ets.delete(@unspent_output_index_table, address) + + InputsWriter.stop(pid) end @doc """ @@ -162,35 +163,28 @@ defmodule Archethic.Account.MemTables.UCOLedger do """ @spec get_inputs(binary()) :: list(VersionedTransactionInput.t()) def get_inputs(address) when is_binary(address) do - unspent = - @unspent_output_index_table - |> :ets.lookup(address) - |> Enum.reduce([], fn {_, from}, acc -> - case :ets.lookup(@ledger_table, {address, from}) do - [] -> - # the output has been spent and is now written on disk - acc - - [{_, amount, spent?, timestamp, reward?, protocol_version}] -> - [ - %VersionedTransactionInput{ - input: %TransactionInput{ - from: from, - amount: amount, - spent?: spent?, - type: :UCO, - timestamp: timestamp, - reward?: reward? - }, - protocol_version: protocol_version - } - | acc - ] - end - end) - |> Enum.reverse() - - spent = Inputs.get_inputs(:UCO, address) - Enum.concat(spent, unspent) + spent = InputsReader.get_inputs(:UCO, address) + + @unspent_output_index_table + |> :ets.lookup(address) + |> Enum.reduce(spent, fn {_, from}, acc -> + [{_, amount, spent?, timestamp, reward?, protocol_version}] = + :ets.lookup(@ledger_table, {address, from}) + + [ + %VersionedTransactionInput{ + input: %TransactionInput{ + from: from, + amount: amount, + spent?: spent?, + type: :UCO, + timestamp: timestamp, + reward?: reward? + }, + protocol_version: protocol_version + } + | acc + ] + end) end end diff --git a/lib/archethic/db/embedded_impl/inputs.ex b/lib/archethic/db/embedded_impl/inputs.ex deleted file mode 100644 index 76af44104..000000000 --- a/lib/archethic/db/embedded_impl/inputs.ex +++ /dev/null @@ -1,73 +0,0 @@ -defmodule Archethic.DB.EmbeddedImpl.Inputs do - @moduledoc """ - Inputs are stored by destination address. 1 file per address - There will be many files - """ - - alias Archethic.DB.EmbeddedImpl - alias Archethic.TransactionChain.VersionedTransactionInput - alias Archethic.Utils - - @type ledger :: :token | :UCO - - @spec append_inputs( - ledger :: ledger, - inputs :: list(VersionedTransactionInput.t()), - address :: binary() - ) :: :ok - def append_inputs(ledger, inputs, address) do - filename = address_to_filename(address, ledger) - :ok = File.mkdir_p!(Path.dirname(filename)) - write_inputs_to_file(filename, inputs) - end - - @spec get_inputs(ledger :: ledger, address :: binary()) :: list(VersionedTransactionInput.t()) - def get_inputs(ledger, address) do - address - |> address_to_filename(ledger) - |> read_inputs_from_file() - end - - defp read_inputs_from_file(filename) do - case File.read(filename) do - {:ok, bin} -> - Enum.reverse(deserialize_inputs_file(bin, [])) - - _ -> - [] - end - end - - defp write_inputs_to_file(filename, inputs) do - inputs_bin = - inputs - |> Enum.map(&VersionedTransactionInput.serialize(&1)) - |> :erlang.list_to_bitstring() - |> Utils.wrap_binary() - - File.write!(filename, inputs_bin, [:append, :binary]) - end - - # instead of ignoring padding, we should count the iteration and stop once over - defp deserialize_inputs_file(bin, acc) do - if bit_size(bin) < 8 do - # less than a byte, we are in the padding of wrap_binary - acc - else - # todo: ORDER? - # deserialize ONE input and return the rest - {input, rest} = VersionedTransactionInput.deserialize(bin) - deserialize_inputs_file(rest, [input | acc]) - end - end - - defp address_to_filename(address, ledger) do - prefix = - case ledger do - :UCO -> "uco" - :token -> "token" - end - - Path.join([EmbeddedImpl.db_path(), "inputs", prefix, Base.encode16(address)]) - end -end diff --git a/lib/archethic/db/embedded_impl/inputs_reader.ex b/lib/archethic/db/embedded_impl/inputs_reader.ex new file mode 100644 index 000000000..70697a812 --- /dev/null +++ b/lib/archethic/db/embedded_impl/inputs_reader.ex @@ -0,0 +1,54 @@ +defmodule Archethic.DB.EmbeddedImpl.InputsReader do + @moduledoc """ + Inputs are stored by destination address. 1 file per address per ledger + """ + + alias Archethic.DB.EmbeddedImpl.InputsWriter + alias Archethic.TransactionChain.VersionedTransactionInput + alias Archethic.Utils + + @spec get_inputs(ledger :: InputsWriter.ledger(), address :: binary()) :: + list(VersionedTransactionInput.t()) + def get_inputs(ledger, address) do + filename = InputsWriter.address_to_filename(ledger, address) + + case File.open(filename, [:read, :binary]) do + {:error, :enoent} -> + [] + + {:ok, fd} -> + case IO.binread(fd, :eof) do + :eof -> + [] + + bin -> + bin + |> deserialize_inputs_file([]) + |> Enum.reverse() + end + end + end + + defp deserialize_inputs_file(<<>>, acc), do: acc + + defp deserialize_inputs_file(bitstring, acc) do + {input_bit_size, rest} = Utils.VarInt.get_value(bitstring) + + # every serialization contains some padding to be a binary (multipe of 8bits) + {input_bitstring, rest} = + case rem(input_bit_size, 8) do + 0 -> + <> = rest + {input_bitstring, rest} + + remainder -> + <> = rest + + {input_bitstring, rest} + end + + {input, <<>>} = VersionedTransactionInput.deserialize(input_bitstring) + deserialize_inputs_file(rest, [input | acc]) + end +end diff --git a/lib/archethic/db/embedded_impl/inputs_writer.ex b/lib/archethic/db/embedded_impl/inputs_writer.ex new file mode 100644 index 000000000..9216a9041 --- /dev/null +++ b/lib/archethic/db/embedded_impl/inputs_writer.ex @@ -0,0 +1,66 @@ +defmodule Archethic.DB.EmbeddedImpl.InputsWriter do + @moduledoc """ + Inputs are stored by destination address. 1 file per address + There will be many files + """ + use GenServer + + alias Archethic.DB.EmbeddedImpl + alias Archethic.TransactionChain.VersionedTransactionInput + alias Archethic.Utils + + @type ledger :: :token | :UCO + + @spec start_link(ledger :: ledger, address :: binary()) :: {:ok, pid()} + def start_link(ledger, address) do + GenServer.start_link(__MODULE__, ledger: ledger, address: address) + end + + @spec stop(pid :: pid()) :: :ok + def stop(pid) do + GenServer.stop(pid) + end + + @spec append_input( + pid :: pid(), + input :: VersionedTransactionInput.t() + ) :: :ok + def append_input(pid, input) do + GenServer.call(pid, {:append_input, input}) + end + + @spec address_to_filename(ledger :: ledger, address :: binary()) :: String.t() + def address_to_filename(ledger, address) do + prefix = + case ledger do + :UCO -> "uco" + :token -> "token" + end + + Path.join([EmbeddedImpl.db_path(), "inputs", prefix, Base.encode16(address)]) + end + + def init(opts) do + ledger = Keyword.get(opts, :ledger) + address = Keyword.get(opts, :address) + filename = address_to_filename(ledger, address) + :ok = File.mkdir_p!(Path.dirname(filename)) + + {:ok, %{filename: filename, fd: File.open!(filename, [:read, :append, :binary])}} + end + + def terminate(_reason, %{fd: fd}) do + File.close(fd) + end + + def handle_call({:append_input, input}, _from, state = %{fd: fd}) do + # we need to pad the bitstring to be a binary + # we also need to prefix with the number of bits to be able to ignore padding to deserialize + serialized_input = VersionedTransactionInput.serialize(input) + encoded_size = Utils.VarInt.from_value(bit_size(serialized_input)) + wrapped_bin = Utils.wrap_binary(<>) + + IO.binwrite(fd, wrapped_bin) + {:reply, :ok, state} + end +end diff --git a/test/archethic/account/mem_tables/token_ledger_test.exs b/test/archethic/account/mem_tables/token_ledger_test.exs index 74699c888..41f624825 100644 --- a/test/archethic/account/mem_tables/token_ledger_test.exs +++ b/test/archethic/account/mem_tables/token_ledger_test.exs @@ -207,28 +207,39 @@ defmodule Archethic.Account.MemTables.TokenLedgerTest do } ) - assert [ - %VersionedTransactionInput{ - input: %TransactionInput{ - from: ^bob3, - amount: 300_000_000, - type: {:token, @token1, 0}, - spent?: false, - timestamp: ~U[2022-10-10 09:27:17Z] - }, - protocol_version: 1 - }, - %VersionedTransactionInput{ - input: %TransactionInput{ - from: ^charlie10, - amount: 100_000_000, - type: {:token, @token1, 1}, - spent?: false, - timestamp: ~U[2022-10-10 09:27:17Z] - }, - protocol_version: 1 - } - ] = TokenLedger.get_inputs(alice2) + # cannot rely on ordering because ETS are ordered by key and here the keys are randomly generated + inputs = TokenLedger.get_inputs(alice2) + assert length(inputs) == 2 + + assert Enum.any?( + inputs, + &(&1 == + %VersionedTransactionInput{ + input: %TransactionInput{ + from: bob3, + amount: 300_000_000, + type: {:token, @token1, 0}, + spent?: false, + timestamp: ~U[2022-10-10 09:27:17Z] + }, + protocol_version: 1 + }) + ) + + assert Enum.any?( + inputs, + &(&1 == + %VersionedTransactionInput{ + input: %TransactionInput{ + from: charlie10, + amount: 100_000_000, + type: {:token, @token1, 1}, + spent?: false, + timestamp: ~U[2022-10-10 09:27:17Z] + }, + protocol_version: 1 + }) + ) end test "should convert spent outputs" do @@ -268,28 +279,39 @@ defmodule Archethic.Account.MemTables.TokenLedgerTest do :ok = TokenLedger.spend_all_unspent_outputs(alice2) - assert [ - %VersionedTransactionInput{ - input: %TransactionInput{ - from: ^bob3, - amount: 300_000_000, - type: {:token, @token1, 0}, - spent?: true, - timestamp: ~U[2022-10-10 09:27:17Z] - }, - protocol_version: 1 - }, - %VersionedTransactionInput{ - input: %TransactionInput{ - from: ^charlie10, - amount: 100_000_000, - type: {:token, @token1, 1}, - spent?: true, - timestamp: ~U[2022-10-10 09:27:17Z] - }, - protocol_version: 1 - } - ] = TokenLedger.get_inputs(alice2) + # cannot rely on ordering because ETS are ordered by key and here the keys are randomly generated + inputs = TokenLedger.get_inputs(alice2) + assert length(inputs) == 2 + + assert Enum.any?( + inputs, + &(&1 == + %VersionedTransactionInput{ + input: %TransactionInput{ + from: bob3, + amount: 300_000_000, + type: {:token, @token1, 0}, + spent?: true, + timestamp: ~U[2022-10-10 09:27:17Z] + }, + protocol_version: 1 + }) + ) + + assert Enum.any?( + inputs, + &(&1 == + %VersionedTransactionInput{ + input: %TransactionInput{ + from: charlie10, + amount: 100_000_000, + type: {:token, @token1, 1}, + spent?: true, + timestamp: ~U[2022-10-10 09:27:17Z] + }, + protocol_version: 1 + }) + ) end end end diff --git a/test/archethic/account/mem_tables/uco_ledger_test.exs b/test/archethic/account/mem_tables/uco_ledger_test.exs index 5c3d9658c..0f1e21298 100644 --- a/test/archethic/account/mem_tables/uco_ledger_test.exs +++ b/test/archethic/account/mem_tables/uco_ledger_test.exs @@ -209,28 +209,39 @@ defmodule Archethic.Account.MemTables.UCOLedgerTest do } ) - assert [ - %VersionedTransactionInput{ - input: %TransactionInput{ - from: ^bob3, - amount: 300_000_000, - spent?: false, - type: :UCO, - timestamp: ~U[2022-10-11 09:24:01Z] - }, - protocol_version: 1 - }, - %VersionedTransactionInput{ - input: %TransactionInput{ - from: ^charlie10, - amount: 100_000_000, - spent?: false, - type: :UCO, - timestamp: ~U[2022-10-11 09:24:01Z] - }, - protocol_version: 1 - } - ] = UCOLedger.get_inputs(alice2) + # cannot rely on ordering because ETS are ordered by key and here the keys are randomly generated + inputs = UCOLedger.get_inputs(alice2) + assert length(inputs) == 2 + + assert Enum.any?( + inputs, + &(&1 == + %VersionedTransactionInput{ + input: %TransactionInput{ + from: bob3, + amount: 300_000_000, + spent?: false, + type: :UCO, + timestamp: ~U[2022-10-11 09:24:01Z] + }, + protocol_version: 1 + }) + ) + + assert Enum.any?( + inputs, + &(&1 == + %VersionedTransactionInput{ + input: %TransactionInput{ + from: charlie10, + amount: 100_000_000, + spent?: false, + type: :UCO, + timestamp: ~U[2022-10-11 09:24:01Z] + }, + protocol_version: 1 + }) + ) end test "should convert spent outputs" do @@ -270,28 +281,39 @@ defmodule Archethic.Account.MemTables.UCOLedgerTest do :ok = UCOLedger.spend_all_unspent_outputs(alice2) - assert [ - %VersionedTransactionInput{ - input: %TransactionInput{ - from: ^bob3, - amount: 300_000_000, - spent?: true, - type: :UCO, - timestamp: ~U[2022-10-11 09:24:01Z] - }, - protocol_version: 1 - }, - %VersionedTransactionInput{ - input: %TransactionInput{ - from: ^charlie10, - amount: 100_000_000, - spent?: true, - type: :UCO, - timestamp: ~U[2022-10-11 09:24:01Z] - }, - protocol_version: 1 - } - ] = UCOLedger.get_inputs(alice2) + # cannot rely on ordering because ETS are ordered by key and here the keys are randomly generated + inputs = UCOLedger.get_inputs(alice2) + assert length(inputs) == 2 + + assert Enum.any?( + inputs, + &(&1 == + %VersionedTransactionInput{ + input: %TransactionInput{ + from: bob3, + amount: 300_000_000, + spent?: true, + type: :UCO, + timestamp: ~U[2022-10-11 09:24:01Z] + }, + protocol_version: 1 + }) + ) + + assert Enum.any?( + inputs, + &(&1 == + %VersionedTransactionInput{ + input: %TransactionInput{ + from: charlie10, + amount: 100_000_000, + spent?: true, + type: :UCO, + timestamp: ~U[2022-10-11 09:24:01Z] + }, + protocol_version: 1 + }) + ) end end end diff --git a/test/archethic/db/embedded_impl/inputs_test.exs b/test/archethic/db/embedded_impl/inputs_test.exs index a16d085ef..ff22b7758 100644 --- a/test/archethic/db/embedded_impl/inputs_test.exs +++ b/test/archethic/db/embedded_impl/inputs_test.exs @@ -1,28 +1,18 @@ defmodule Archethic.DB.EmbeddedImpl.InputsTest do use ArchethicCase - alias Archethic.DB.EmbeddedImpl.Inputs + alias Archethic.DB.EmbeddedImpl.InputsReader + alias Archethic.DB.EmbeddedImpl.InputsWriter alias Archethic.TransactionChain.VersionedTransactionInput alias Archethic.TransactionChain.TransactionInput - setup do - db_path = Application.app_dir(:archethic, "data_test") - File.mkdir_p!(db_path) - - on_exit(fn -> - File.rm_rf!(db_path) - end) - - %{db_path: db_path} - end - describe "Append/Get" do test "returns empty when there is none" do address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> - assert [] = Inputs.get_inputs(:UCO, address) - assert [] = Inputs.get_inputs(:token, address) + assert [] = InputsReader.get_inputs(:UCO, address) + assert [] = InputsReader.get_inputs(:token, address) end test "returns the UCO inputs that were appended" do @@ -55,10 +45,12 @@ defmodule Archethic.DB.EmbeddedImpl.InputsTest do } ] - Inputs.append_inputs(:UCO, inputs, address) - assert ^inputs = Inputs.get_inputs(:UCO, address) - assert [] = Inputs.get_inputs(:UCO, address2) - assert [] = Inputs.get_inputs(:UCO, address3) + {:ok, pid1} = InputsWriter.start_link(:UCO, address) + Enum.each(inputs, &InputsWriter.append_input(pid1, &1)) + + assert ^inputs = InputsReader.get_inputs(:UCO, address) + assert [] = InputsReader.get_inputs(:UCO, address2) + assert [] = InputsReader.get_inputs(:UCO, address3) end test "returns the TOKEN inputs that were appended" do @@ -92,10 +84,12 @@ defmodule Archethic.DB.EmbeddedImpl.InputsTest do } ] - Inputs.append_inputs(:token, inputs, address) - assert ^inputs = Inputs.get_inputs(:token, address) - assert [] = Inputs.get_inputs(:token, address2) - assert [] = Inputs.get_inputs(:token, address3) + {:ok, pid1} = InputsWriter.start_link(:token, address) + Enum.each(inputs, &InputsWriter.append_input(pid1, &1)) + + assert ^inputs = InputsReader.get_inputs(:token, address) + assert [] = InputsReader.get_inputs(:token, address2) + assert [] = InputsReader.get_inputs(:token, address3) end end end From c0dfc484d367720721a28fbbe8cb3fb14b3cf502 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Tue, 15 Nov 2022 14:57:52 +0100 Subject: [PATCH 07/19] Move test to the correct test file --- .../transaction/input_test.exs | 22 +++++++++++++++ .../transaction_input_test.exs | 28 ------------------- 2 files changed, 22 insertions(+), 28 deletions(-) delete mode 100644 test/archethic/transaction_chain/transaction_input_test.exs diff --git a/test/archethic/transaction_chain/transaction/input_test.exs b/test/archethic/transaction_chain/transaction/input_test.exs index ac9446bf5..ec84dba24 100644 --- a/test/archethic/transaction_chain/transaction/input_test.exs +++ b/test/archethic/transaction_chain/transaction/input_test.exs @@ -3,6 +3,28 @@ defmodule Archethic.TransactionChain.TransactionInputTest do import ArchethicCase, only: [current_protocol_version: 0] + alias Archethic.Mining alias Archethic.TransactionChain.TransactionInput doctest TransactionInput + + describe "serialization/deserialization workflow" do + test "should return the same transaction after serialization and deserialization" do + input = %TransactionInput{ + amount: 1, + type: :UCO, + from: <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>>, + reward?: true, + spent?: true, + timestamp: DateTime.utc_now() |> DateTime.truncate(:second) + } + + protocol_version = Mining.protocol_version() + + assert {^input, _} = + TransactionInput.deserialize( + TransactionInput.serialize(input, protocol_version), + protocol_version + ) + end + end end diff --git a/test/archethic/transaction_chain/transaction_input_test.exs b/test/archethic/transaction_chain/transaction_input_test.exs deleted file mode 100644 index 5aa0220f3..000000000 --- a/test/archethic/transaction_chain/transaction_input_test.exs +++ /dev/null @@ -1,28 +0,0 @@ -defmodule Archethic.TransactionChain.TransactionInputTest do - use ExUnit.Case - - alias Archethic.Mining - - alias Archethic.TransactionChain.TransactionInput - - describe "serialization/deserialization workflow" do - test "should return the same transaction after serialization and deserialization" do - input = %TransactionInput{ - amount: 1, - type: :UCO, - from: <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>>, - reward?: true, - spent?: true, - timestamp: DateTime.utc_now() |> DateTime.truncate(:second) - } - - protocol_version = Mining.protocol_version() - - assert {^input, _} = - TransactionInput.deserialize( - TransactionInput.serialize(input, protocol_version), - protocol_version - ) - end - end -end From 0039f5d859fdd6c059d0e7ea68e681e34c31a832 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Tue, 15 Nov 2022 15:22:41 +0100 Subject: [PATCH 08/19] Use behaviour in DB for the inputs --- .../account/mem_tables/token_ledger.ex | 11 ++++--- .../account/mem_tables/uco_ledger.ex | 11 ++++--- lib/archethic/db.ex | 8 +++++ lib/archethic/db/embedded_impl.ex | 29 +++++++++++++++++++ 4 files changed, 47 insertions(+), 12 deletions(-) diff --git a/lib/archethic/account/mem_tables/token_ledger.ex b/lib/archethic/account/mem_tables/token_ledger.ex index 4614b7dad..59c0cf262 100644 --- a/lib/archethic/account/mem_tables/token_ledger.ex +++ b/lib/archethic/account/mem_tables/token_ledger.ex @@ -4,8 +4,7 @@ defmodule Archethic.Account.MemTables.TokenLedger do @ledger_table :archethic_token_ledger @unspent_output_index_table :archethic_token_unspent_output_index - alias Archethic.DB.EmbeddedImpl.InputsReader - alias Archethic.DB.EmbeddedImpl.InputsWriter + alias Archethic.DB.EmbeddedImpl alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations.UnspentOutput @@ -136,7 +135,7 @@ defmodule Archethic.Account.MemTables.TokenLedger do """ @spec spend_all_unspent_outputs(binary()) :: :ok def spend_all_unspent_outputs(address) do - {:ok, pid} = InputsWriter.start_link(:token, address) + {:ok, pid} = EmbeddedImpl.start_inputs_writer(:token, address) @unspent_output_index_table |> :ets.lookup(address) @@ -144,7 +143,7 @@ defmodule Archethic.Account.MemTables.TokenLedger do [{_, amount, _, timestamp, protocol_version}] = :ets.lookup(@ledger_table, {to, from, token_address, token_id}) - InputsWriter.append_input(pid, %VersionedTransactionInput{ + EmbeddedImpl.append_input(pid, %VersionedTransactionInput{ protocol_version: protocol_version, input: %TransactionInput{ from: from, @@ -160,7 +159,7 @@ defmodule Archethic.Account.MemTables.TokenLedger do :ets.delete(@unspent_output_index_table, address) - InputsWriter.stop(pid) + EmbeddedImpl.stop_inputs_writer(pid) end @doc """ @@ -168,7 +167,7 @@ defmodule Archethic.Account.MemTables.TokenLedger do """ @spec get_inputs(binary()) :: list(VersionedTransactionInput.t()) def get_inputs(address) when is_binary(address) do - spent = InputsReader.get_inputs(:token, address) + spent = EmbeddedImpl.get_inputs(:token, address) @unspent_output_index_table |> :ets.lookup(address) diff --git a/lib/archethic/account/mem_tables/uco_ledger.ex b/lib/archethic/account/mem_tables/uco_ledger.ex index c1968fd1b..d9913af7e 100644 --- a/lib/archethic/account/mem_tables/uco_ledger.ex +++ b/lib/archethic/account/mem_tables/uco_ledger.ex @@ -4,8 +4,7 @@ defmodule Archethic.Account.MemTables.UCOLedger do @ledger_table :archethic_uco_ledger @unspent_output_index_table :archethic_uco_unspent_output_index - alias Archethic.DB.EmbeddedImpl.InputsReader - alias Archethic.DB.EmbeddedImpl.InputsWriter + alias Archethic.DB.EmbeddedImpl alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations.UnspentOutput @@ -130,7 +129,7 @@ defmodule Archethic.Account.MemTables.UCOLedger do """ @spec spend_all_unspent_outputs(binary()) :: :ok def spend_all_unspent_outputs(address) do - {:ok, pid} = InputsWriter.start_link(:UCO, address) + {:ok, pid} = EmbeddedImpl.start_inputs_writer(:UCO, address) @unspent_output_index_table |> :ets.lookup(address) @@ -138,7 +137,7 @@ defmodule Archethic.Account.MemTables.UCOLedger do [{_, amount, _, timestamp, reward?, protocol_version}] = :ets.lookup(@ledger_table, {to, from}) - InputsWriter.append_input(pid, %VersionedTransactionInput{ + EmbeddedImpl.append_input(pid, %VersionedTransactionInput{ protocol_version: protocol_version, input: %TransactionInput{ from: from, @@ -155,7 +154,7 @@ defmodule Archethic.Account.MemTables.UCOLedger do :ets.delete(@unspent_output_index_table, address) - InputsWriter.stop(pid) + EmbeddedImpl.stop_inputs_writer(pid) end @doc """ @@ -163,7 +162,7 @@ defmodule Archethic.Account.MemTables.UCOLedger do """ @spec get_inputs(binary()) :: list(VersionedTransactionInput.t()) def get_inputs(address) when is_binary(address) do - spent = InputsReader.get_inputs(:UCO, address) + spent = EmbeddedImpl.get_inputs(:UCO, address) @unspent_output_index_table |> :ets.lookup(address) diff --git a/lib/archethic/db.ex b/lib/archethic/db.ex index 535798dfd..d32870812 100644 --- a/lib/archethic/db.ex +++ b/lib/archethic/db.ex @@ -9,6 +9,7 @@ defmodule Archethic.DB do alias __MODULE__.EmbeddedImpl alias Archethic.TransactionChain.Transaction + alias Archethic.TransactionChain.VersionedTransactionInput use Knigge, otp_app: :archethic, default: EmbeddedImpl @@ -76,4 +77,11 @@ defmodule Archethic.DB do @callback get_bootstrap_info(key :: String.t()) :: String.t() | nil @callback set_bootstrap_info(key :: String.t(), value :: String.t()) :: :ok + + @callback start_inputs_writer(ledger :: :UCO | :token, address :: binary()) :: {:ok, pid()} + @callback stop_inputs_writer(pid :: pid()) :: :ok + @callback append_input(pid :: pid(), VersionedTransactionInput.t()) :: + :ok + @callback get_inputs(ledger :: :UCO | :token, address :: binary()) :: + list(VersionedTransactionInput.t()) end diff --git a/lib/archethic/db/embedded_impl.ex b/lib/archethic/db/embedded_impl.ex index 85838a967..5b1648f48 100644 --- a/lib/archethic/db/embedded_impl.ex +++ b/lib/archethic/db/embedded_impl.ex @@ -13,10 +13,13 @@ defmodule Archethic.DB.EmbeddedImpl do alias __MODULE__.ChainIndex alias __MODULE__.ChainReader alias __MODULE__.ChainWriter + alias __MODULE__.InputsReader + alias __MODULE__.InputsWriter alias __MODULE__.P2PView alias __MODULE__.StatsInfo alias Archethic.TransactionChain.Transaction + alias Archethic.TransactionChain.VersionedTransactionInput alias Archethic.Utils @@ -366,4 +369,30 @@ defmodule Archethic.DB.EmbeddedImpl do def scan_chain(genesis_address, limit_address, fields \\ [], paging_state \\ nil) do ChainReader.scan_chain(genesis_address, limit_address, fields, paging_state, db_path()) end + + @doc """ + Start a process responsible to write the inputs + """ + @spec start_inputs_writer(ledger :: :UCO | :token, address :: binary()) :: {:ok, pid()} + defdelegate start_inputs_writer(ledger, address), to: InputsWriter, as: :start_link + + @doc """ + Stop the process responsible to write the inputs + """ + @spec stop_inputs_writer(pid :: pid()) :: :ok + defdelegate stop_inputs_writer(pid), to: InputsWriter, as: :stop + + @doc """ + Appends one input to existing inputs + """ + @spec append_input(pid :: pid(), VersionedTransactionInput.t()) :: + :ok + defdelegate append_input(pid, input), to: InputsWriter, as: :append_input + + @doc """ + Read the list of inputs available at address + """ + @spec get_inputs(ledger :: :UCO | :token, address :: binary()) :: + list(VersionedTransactionInput.t()) + defdelegate get_inputs(ledger, address), to: InputsReader, as: :get_inputs end From dd6c2db01ee03afe5020db782e05aac9d259b1e3 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Wed, 16 Nov 2022 10:32:53 +0100 Subject: [PATCH 09/19] Ledgers use the DB module directly instead of the embedded impl --- .../account/mem_tables/token_ledger.ex | 10 ++-- .../account/mem_tables/uco_ledger.ex | 10 ++-- .../account/mem_tables/token_ledger_test.exs | 42 ++++++++++++++++ .../account/mem_tables/uco_ledger_test.exs | 42 ++++++++++++++++ test/archethic/account_test.exs | 49 +++++++++++++++++++ test/support/template.ex | 4 ++ 6 files changed, 147 insertions(+), 10 deletions(-) diff --git a/lib/archethic/account/mem_tables/token_ledger.ex b/lib/archethic/account/mem_tables/token_ledger.ex index 59c0cf262..71049c1f8 100644 --- a/lib/archethic/account/mem_tables/token_ledger.ex +++ b/lib/archethic/account/mem_tables/token_ledger.ex @@ -4,7 +4,7 @@ defmodule Archethic.Account.MemTables.TokenLedger do @ledger_table :archethic_token_ledger @unspent_output_index_table :archethic_token_unspent_output_index - alias Archethic.DB.EmbeddedImpl + alias Archethic.DB alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations.UnspentOutput @@ -135,7 +135,7 @@ defmodule Archethic.Account.MemTables.TokenLedger do """ @spec spend_all_unspent_outputs(binary()) :: :ok def spend_all_unspent_outputs(address) do - {:ok, pid} = EmbeddedImpl.start_inputs_writer(:token, address) + {:ok, pid} = DB.start_inputs_writer(:token, address) @unspent_output_index_table |> :ets.lookup(address) @@ -143,7 +143,7 @@ defmodule Archethic.Account.MemTables.TokenLedger do [{_, amount, _, timestamp, protocol_version}] = :ets.lookup(@ledger_table, {to, from, token_address, token_id}) - EmbeddedImpl.append_input(pid, %VersionedTransactionInput{ + DB.append_input(pid, %VersionedTransactionInput{ protocol_version: protocol_version, input: %TransactionInput{ from: from, @@ -159,7 +159,7 @@ defmodule Archethic.Account.MemTables.TokenLedger do :ets.delete(@unspent_output_index_table, address) - EmbeddedImpl.stop_inputs_writer(pid) + DB.stop_inputs_writer(pid) end @doc """ @@ -167,7 +167,7 @@ defmodule Archethic.Account.MemTables.TokenLedger do """ @spec get_inputs(binary()) :: list(VersionedTransactionInput.t()) def get_inputs(address) when is_binary(address) do - spent = EmbeddedImpl.get_inputs(:token, address) + spent = DB.get_inputs(:token, address) @unspent_output_index_table |> :ets.lookup(address) diff --git a/lib/archethic/account/mem_tables/uco_ledger.ex b/lib/archethic/account/mem_tables/uco_ledger.ex index d9913af7e..a1f572c4c 100644 --- a/lib/archethic/account/mem_tables/uco_ledger.ex +++ b/lib/archethic/account/mem_tables/uco_ledger.ex @@ -4,7 +4,7 @@ defmodule Archethic.Account.MemTables.UCOLedger do @ledger_table :archethic_uco_ledger @unspent_output_index_table :archethic_uco_unspent_output_index - alias Archethic.DB.EmbeddedImpl + alias Archethic.DB alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations.UnspentOutput @@ -129,7 +129,7 @@ defmodule Archethic.Account.MemTables.UCOLedger do """ @spec spend_all_unspent_outputs(binary()) :: :ok def spend_all_unspent_outputs(address) do - {:ok, pid} = EmbeddedImpl.start_inputs_writer(:UCO, address) + {:ok, pid} = DB.start_inputs_writer(:UCO, address) @unspent_output_index_table |> :ets.lookup(address) @@ -137,7 +137,7 @@ defmodule Archethic.Account.MemTables.UCOLedger do [{_, amount, _, timestamp, reward?, protocol_version}] = :ets.lookup(@ledger_table, {to, from}) - EmbeddedImpl.append_input(pid, %VersionedTransactionInput{ + DB.append_input(pid, %VersionedTransactionInput{ protocol_version: protocol_version, input: %TransactionInput{ from: from, @@ -154,7 +154,7 @@ defmodule Archethic.Account.MemTables.UCOLedger do :ets.delete(@unspent_output_index_table, address) - EmbeddedImpl.stop_inputs_writer(pid) + DB.stop_inputs_writer(pid) end @doc """ @@ -162,7 +162,7 @@ defmodule Archethic.Account.MemTables.UCOLedger do """ @spec get_inputs(binary()) :: list(VersionedTransactionInput.t()) def get_inputs(address) when is_binary(address) do - spent = EmbeddedImpl.get_inputs(:UCO, address) + spent = DB.get_inputs(:UCO, address) @unspent_output_index_table |> :ets.lookup(address) diff --git a/test/archethic/account/mem_tables/token_ledger_test.exs b/test/archethic/account/mem_tables/token_ledger_test.exs index 41f624825..286b75736 100644 --- a/test/archethic/account/mem_tables/token_ledger_test.exs +++ b/test/archethic/account/mem_tables/token_ledger_test.exs @@ -13,6 +13,8 @@ defmodule Archethic.Account.MemTables.TokenLedgerTest do alias Archethic.TransactionChain.TransactionInput alias Archethic.TransactionChain.VersionedTransactionInput + import Mox + test "add_unspent_output/3 insert a new entry in the tables" do {:ok, _pid} = TokenLedger.start_link() @@ -166,6 +168,11 @@ defmodule Archethic.Account.MemTables.TokenLedgerTest do protocol_version: 1 }) + MockDB + |> expect(:start_inputs_writer, fn _, _ -> {:ok, self()} end) + |> expect(:stop_inputs_writer, fn _ -> :ok end) + |> stub(:append_input, fn _, _ -> :ok end) + :ok = TokenLedger.spend_all_unspent_outputs(alice2) assert [] = TokenLedger.get_unspent_outputs(alice2) @@ -207,6 +214,12 @@ defmodule Archethic.Account.MemTables.TokenLedgerTest do } ) + MockDB + |> expect(:start_inputs_writer, fn _, _ -> {:ok, self()} end) + |> expect(:stop_inputs_writer, fn _ -> :ok end) + |> stub(:append_input, fn _, _ -> :ok end) + |> expect(:get_inputs, fn _, _ -> [] end) + # cannot rely on ordering because ETS are ordered by key and here the keys are randomly generated inputs = TokenLedger.get_inputs(alice2) assert length(inputs) == 2 @@ -277,6 +290,35 @@ defmodule Archethic.Account.MemTables.TokenLedgerTest do } ) + MockDB + |> expect(:start_inputs_writer, fn _, _ -> {:ok, self()} end) + |> expect(:stop_inputs_writer, fn _ -> :ok end) + |> stub(:append_input, fn _, _ -> :ok end) + |> expect(:get_inputs, fn _, _ -> + [ + %VersionedTransactionInput{ + input: %TransactionInput{ + spent?: true, + from: bob3, + amount: 300_000_000, + type: {:token, @token1, 0}, + timestamp: ~U[2022-10-10 09:27:17Z] + }, + protocol_version: 1 + }, + %VersionedTransactionInput{ + input: %TransactionInput{ + spent?: true, + from: charlie10, + amount: 100_000_000, + type: {:token, @token1, 1}, + timestamp: ~U[2022-10-10 09:27:17Z] + }, + protocol_version: 1 + } + ] + end) + :ok = TokenLedger.spend_all_unspent_outputs(alice2) # cannot rely on ordering because ETS are ordered by key and here the keys are randomly generated diff --git a/test/archethic/account/mem_tables/uco_ledger_test.exs b/test/archethic/account/mem_tables/uco_ledger_test.exs index 0f1e21298..115a34302 100644 --- a/test/archethic/account/mem_tables/uco_ledger_test.exs +++ b/test/archethic/account/mem_tables/uco_ledger_test.exs @@ -10,6 +10,8 @@ defmodule Archethic.Account.MemTables.UCOLedgerTest do alias Archethic.TransactionChain.TransactionInput alias Archethic.TransactionChain.VersionedTransactionInput + import Mox + test "add_unspent_output/3 should insert a new entry in the tables" do {:ok, _pid} = UCOLedger.start_link() @@ -169,6 +171,11 @@ defmodule Archethic.Account.MemTables.UCOLedgerTest do } ) + MockDB + |> expect(:start_inputs_writer, fn _, _ -> {:ok, self()} end) + |> expect(:stop_inputs_writer, fn _ -> :ok end) + |> stub(:append_input, fn _, _ -> :ok end) + :ok = UCOLedger.spend_all_unspent_outputs(alice2) assert [] = UCOLedger.get_unspent_outputs(alice2) end @@ -209,6 +216,12 @@ defmodule Archethic.Account.MemTables.UCOLedgerTest do } ) + MockDB + |> expect(:start_inputs_writer, fn _, _ -> {:ok, self()} end) + |> expect(:stop_inputs_writer, fn _ -> :ok end) + |> stub(:append_input, fn _, _ -> :ok end) + |> expect(:get_inputs, fn _, _ -> [] end) + # cannot rely on ordering because ETS are ordered by key and here the keys are randomly generated inputs = UCOLedger.get_inputs(alice2) assert length(inputs) == 2 @@ -279,6 +292,35 @@ defmodule Archethic.Account.MemTables.UCOLedgerTest do } ) + MockDB + |> expect(:start_inputs_writer, fn _, _ -> {:ok, self()} end) + |> expect(:stop_inputs_writer, fn _ -> :ok end) + |> stub(:append_input, fn _, _ -> :ok end) + |> expect(:get_inputs, fn _, _ -> + [ + %VersionedTransactionInput{ + input: %TransactionInput{ + from: bob3, + amount: 300_000_000, + spent?: true, + type: :UCO, + timestamp: ~U[2022-10-11 09:24:01Z] + }, + protocol_version: 1 + }, + %VersionedTransactionInput{ + input: %TransactionInput{ + from: charlie10, + amount: 100_000_000, + spent?: true, + type: :UCO, + timestamp: ~U[2022-10-11 09:24:01Z] + }, + protocol_version: 1 + } + ] + end) + :ok = UCOLedger.spend_all_unspent_outputs(alice2) # cannot rely on ordering because ETS are ordered by key and here the keys are randomly generated diff --git a/test/archethic/account_test.exs b/test/archethic/account_test.exs index d8222ce20..0e6cce97d 100644 --- a/test/archethic/account_test.exs +++ b/test/archethic/account_test.exs @@ -173,6 +173,11 @@ defmodule Archethic.AccountTest do } ) + MockDB + |> stub(:start_inputs_writer, fn _, _ -> {:ok, self()} end) + |> stub(:stop_inputs_writer, fn _ -> :ok end) + |> stub(:append_input, fn _, _ -> :ok end) + UCOLedger.spend_all_unspent_outputs(alice2_address) TokenLedger.spend_all_unspent_outputs(alice2_address) @@ -229,6 +234,50 @@ defmodule Archethic.AccountTest do } ) + MockDB + |> stub(:start_inputs_writer, fn _, _ -> {:ok, self()} end) + |> stub(:stop_inputs_writer, fn _ -> :ok end) + |> stub(:append_input, fn _, _ -> :ok end) + |> stub(:get_inputs, fn + :UCO, _ -> + [ + %VersionedTransactionInput{ + input: %TransactionInput{ + from: tom10_address, + amount: 100_000_000, + type: :UCO, + timestamp: timestamp, + spent?: true + }, + protocol_version: 1 + }, + %VersionedTransactionInput{ + input: %TransactionInput{ + from: bob3_address, + amount: 300_000_000, + type: :UCO, + timestamp: timestamp, + spent?: true + }, + protocol_version: 1 + } + ] + + :token, _ -> + [ + %VersionedTransactionInput{ + input: %TransactionInput{ + from: charlie2_address, + amount: 10_000_000_000, + type: {:token, charlie_token_address, 0}, + timestamp: timestamp, + spent?: true + }, + protocol_version: 1 + } + ] + end) + UCOLedger.spend_all_unspent_outputs(alice2_address) TokenLedger.spend_all_unspent_outputs(alice2_address) diff --git a/test/support/template.ex b/test/support/template.ex index 983f8dc77..4c9341a1c 100644 --- a/test/support/template.ex +++ b/test/support/template.ex @@ -80,6 +80,10 @@ defmodule ArchethicCase do |> stub(:clear_beacon_summaries, fn -> :ok end) |> stub(:get_beacon_summary, fn _ -> {:error, :not_exists} end) |> stub(:get_last_chain_public_key, fn public_key, _ -> public_key end) + |> stub(:start_inputs_writer, fn _, _ -> {:ok, self()} end) + |> stub(:stop_inputs_writer, fn _ -> :ok end) + |> stub(:append_input, fn _, _ -> :ok end) + |> stub(:get_inputs, fn _, _ -> [] end) {:ok, shared_secrets_counter} = Agent.start_link(fn -> 0 end) {:ok, network_pool_counter} = Agent.start_link(fn -> 0 end) From c8eb0d6b94b4934f48adb78a53b216a2c7516d7d Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Wed, 16 Nov 2022 16:45:15 +0100 Subject: [PATCH 10/19] Avoid inputs duplication on multiple calls --- .../db/embedded_impl/inputs_writer.ex | 17 +++++++++-- .../db/embedded_impl/inputs_test.exs | 29 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/lib/archethic/db/embedded_impl/inputs_writer.ex b/lib/archethic/db/embedded_impl/inputs_writer.ex index 9216a9041..c15f4e8fb 100644 --- a/lib/archethic/db/embedded_impl/inputs_writer.ex +++ b/lib/archethic/db/embedded_impl/inputs_writer.ex @@ -1,6 +1,6 @@ defmodule Archethic.DB.EmbeddedImpl.InputsWriter do @moduledoc """ - Inputs are stored by destination address. 1 file per address + Inputs are stored by destination address. 1 file per address per ledger There will be many files """ use GenServer @@ -46,7 +46,20 @@ defmodule Archethic.DB.EmbeddedImpl.InputsWriter do filename = address_to_filename(ledger, address) :ok = File.mkdir_p!(Path.dirname(filename)) - {:ok, %{filename: filename, fd: File.open!(filename, [:read, :append, :binary])}} + # We use the `exclusive` flag. This means we can only open a file that does not exist yet. + # We use this mechanism to prevent rewriting the same data over and over when we restart the node and reprocess the transaction history. + # If the InputsWriter is called on an existing file, it will behave as normal but will write to the null device (= do nothing) + # This optimization is possible only because we always spend all the inputs of an address at the same time + fd = + case File.open(filename, [:binary, :exclusive]) do + {:error, :eexist} -> + File.open!("/dev/null", [:binary, :write]) + + {:ok, iodevice} -> + iodevice + end + + {:ok, %{filename: filename, fd: fd}} end def terminate(_reason, %{fd: fd}) do diff --git a/test/archethic/db/embedded_impl/inputs_test.exs b/test/archethic/db/embedded_impl/inputs_test.exs index ff22b7758..9448cded5 100644 --- a/test/archethic/db/embedded_impl/inputs_test.exs +++ b/test/archethic/db/embedded_impl/inputs_test.exs @@ -15,6 +15,35 @@ defmodule Archethic.DB.EmbeddedImpl.InputsTest do assert [] = InputsReader.get_inputs(:token, address) end + test "should not duplicate the inputs if called multiple times" do + # it means that we can only open the file to write *once* + address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + + tx = %VersionedTransactionInput{ + protocol_version: 1, + input: %TransactionInput{ + amount: 1, + type: :UCO, + from: address, + reward?: true, + spent?: true, + timestamp: ~U[2022-11-14 14:54:12Z] + } + } + + {:ok, pid1} = InputsWriter.start_link(:UCO, address) + InputsWriter.append_input(pid1, tx) + InputsWriter.stop(pid1) + + assert [^tx] = InputsReader.get_inputs(:UCO, address) + + {:ok, pid2} = InputsWriter.start_link(:UCO, address) + InputsWriter.append_input(pid2, tx) + InputsWriter.stop(pid2) + + assert [^tx] = InputsReader.get_inputs(:UCO, address) + end + test "returns the UCO inputs that were appended" do address = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> address2 = <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> From 1a275851196f07b7a55474fe9569813c0b6f62d8 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Thu, 17 Nov 2022 11:30:26 +0100 Subject: [PATCH 11/19] Change the order of TransactionChain.list_all/1 to chronological --- lib/archethic/db/embedded_impl.ex | 8 ++--- .../db/embedded_impl/chain_reader.ex | 29 ++++++++++++++++++- lib/archethic/transaction_chain.ex | 2 +- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/lib/archethic/db/embedded_impl.ex b/lib/archethic/db/embedded_impl.ex index 5b1648f48..ea5d240ff 100644 --- a/lib/archethic/db/embedded_impl.ex +++ b/lib/archethic/db/embedded_impl.ex @@ -282,12 +282,8 @@ defmodule Archethic.DB.EmbeddedImpl do """ @spec list_transactions(fields :: list()) :: Enumerable.t() | list(Transaction.t()) def list_transactions(fields \\ []) when is_list(fields) do - db_path() - |> ChainIndex.list_all_addresses() - |> Stream.map(fn address -> - {:ok, tx} = get_transaction(address, fields) - tx - end) + ChainIndex.list_genesis_addresses() + |> Stream.flat_map(&ChainReader.stream_scan_chain(&1, nil, fields, db_path())) end @doc """ diff --git a/lib/archethic/db/embedded_impl/chain_reader.ex b/lib/archethic/db/embedded_impl/chain_reader.ex index 07c556009..b43f6f338 100644 --- a/lib/archethic/db/embedded_impl/chain_reader.ex +++ b/lib/archethic/db/embedded_impl/chain_reader.ex @@ -223,7 +223,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainReader do @spec scan_chain( genesis_address :: binary(), limit_address :: binary(), - list(), + fields :: list(), paging_address :: nil | binary(), db_path :: binary() ) :: @@ -279,6 +279,33 @@ defmodule Archethic.DB.EmbeddedImpl.ChainReader do end end + @doc """ + Stream chain tx from the beginning until a given limit address + """ + @spec stream_scan_chain( + genesis_address :: binary(), + limit_address :: binary(), + fields :: list(), + db_path :: binary() + ) :: Enumerable.t() + def stream_scan_chain(genesis_address, limit_address, fields, db_path) do + Stream.resource( + fn -> scan_chain(genesis_address, limit_address, fields, nil, db_path) end, + fn + {transactions, true, paging_state} -> + {transactions, + scan_chain(genesis_address, limit_address, fields, paging_state, db_path)} + + {transactions, false, _} -> + {transactions, :eof} + + :eof -> + {:halt, nil} + end, + fn _ -> :ok end + ) + end + @doc """ ## Examples diff --git a/lib/archethic/transaction_chain.ex b/lib/archethic/transaction_chain.ex index 65014bf20..e983a1a9e 100644 --- a/lib/archethic/transaction_chain.ex +++ b/lib/archethic/transaction_chain.ex @@ -51,7 +51,7 @@ defmodule Archethic.TransactionChain do require Logger @doc """ - List all the transaction chain stored + List all the transaction chain stored. Chronological order within a transaction chain """ @spec list_all(fields :: list()) :: Enumerable.t() defdelegate list_all(fields \\ []), to: DB, as: :list_transactions From 3a63a1efc31f5e5b4a8846f0367bd7979591f4c7 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Thu, 17 Nov 2022 11:45:02 +0100 Subject: [PATCH 12/19] fix types for dialyzer --- lib/archethic/db/embedded_impl/chain_reader.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/archethic/db/embedded_impl/chain_reader.ex b/lib/archethic/db/embedded_impl/chain_reader.ex index b43f6f338..57cca625f 100644 --- a/lib/archethic/db/embedded_impl/chain_reader.ex +++ b/lib/archethic/db/embedded_impl/chain_reader.ex @@ -222,7 +222,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainReader do """ @spec scan_chain( genesis_address :: binary(), - limit_address :: binary(), + limit_address :: nil | binary(), fields :: list(), paging_address :: nil | binary(), db_path :: binary() @@ -284,7 +284,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainReader do """ @spec stream_scan_chain( genesis_address :: binary(), - limit_address :: binary(), + limit_address :: nil | binary(), fields :: list(), db_path :: binary() ) :: Enumerable.t() From 2beff4a972f10706541dea2e9d12ba7c6d3dac92 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Mon, 21 Nov 2022 15:08:24 +0100 Subject: [PATCH 13/19] UCO ledger optimization UCO are either all spent or all unspent, so no need to read from disk if the ETS table contains spent uco --- .../account/mem_tables/uco_ledger.ex | 54 +++++++++++-------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/lib/archethic/account/mem_tables/uco_ledger.ex b/lib/archethic/account/mem_tables/uco_ledger.ex index a1f572c4c..f88a57e2f 100644 --- a/lib/archethic/account/mem_tables/uco_ledger.ex +++ b/lib/archethic/account/mem_tables/uco_ledger.ex @@ -162,28 +162,36 @@ defmodule Archethic.Account.MemTables.UCOLedger do """ @spec get_inputs(binary()) :: list(VersionedTransactionInput.t()) def get_inputs(address) when is_binary(address) do - spent = DB.get_inputs(:UCO, address) - - @unspent_output_index_table - |> :ets.lookup(address) - |> Enum.reduce(spent, fn {_, from}, acc -> - [{_, amount, spent?, timestamp, reward?, protocol_version}] = - :ets.lookup(@ledger_table, {address, from}) - - [ - %VersionedTransactionInput{ - input: %TransactionInput{ - from: from, - amount: amount, - spent?: spent?, - type: :UCO, - timestamp: timestamp, - reward?: reward? - }, - protocol_version: protocol_version - } - | acc - ] - end) + unspent_inputs = + @unspent_output_index_table + |> :ets.lookup(address) + |> Enum.reduce([], fn {_, from}, acc -> + [{_, amount, spent?, timestamp, reward?, protocol_version}] = + :ets.lookup(@ledger_table, {address, from}) + + [ + %VersionedTransactionInput{ + input: %TransactionInput{ + from: from, + amount: amount, + spent?: spent?, + type: :UCO, + timestamp: timestamp, + reward?: reward? + }, + protocol_version: protocol_version + } + | acc + ] + end) + + # UCO inputs are either all spent or all unspent at a given address + case unspent_inputs do + [] -> + DB.get_inputs(:UCO, address) + + _ -> + unspent_inputs + end end end From 452b79866c477a81a73379b03a9e5313608278af Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Mon, 21 Nov 2022 15:14:20 +0100 Subject: [PATCH 14/19] db: use file.read instead of file.open and io.binread --- lib/archethic/db/embedded_impl/inputs_reader.ex | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/lib/archethic/db/embedded_impl/inputs_reader.ex b/lib/archethic/db/embedded_impl/inputs_reader.ex index 70697a812..253b3bf72 100644 --- a/lib/archethic/db/embedded_impl/inputs_reader.ex +++ b/lib/archethic/db/embedded_impl/inputs_reader.ex @@ -12,20 +12,14 @@ defmodule Archethic.DB.EmbeddedImpl.InputsReader do def get_inputs(ledger, address) do filename = InputsWriter.address_to_filename(ledger, address) - case File.open(filename, [:read, :binary]) do + case File.read(filename) do {:error, :enoent} -> [] - {:ok, fd} -> - case IO.binread(fd, :eof) do - :eof -> - [] - - bin -> - bin - |> deserialize_inputs_file([]) - |> Enum.reverse() - end + {:ok, bin} -> + bin + |> deserialize_inputs_file([]) + |> Enum.reverse() end end From ef01b9392fecdd0f561e0f3d201de652b207e7d3 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Mon, 21 Nov 2022 15:32:06 +0100 Subject: [PATCH 15/19] Token ledger optimization Tokens are either all spent or all unspent, so no need to read from disk if the ETS table contains spent tokens --- .../account/mem_tables/token_ledger.ex | 52 +++++++++++-------- .../account/mem_tables/uco_ledger.ex | 2 +- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/lib/archethic/account/mem_tables/token_ledger.ex b/lib/archethic/account/mem_tables/token_ledger.ex index 71049c1f8..99036afef 100644 --- a/lib/archethic/account/mem_tables/token_ledger.ex +++ b/lib/archethic/account/mem_tables/token_ledger.ex @@ -167,27 +167,35 @@ defmodule Archethic.Account.MemTables.TokenLedger do """ @spec get_inputs(binary()) :: list(VersionedTransactionInput.t()) def get_inputs(address) when is_binary(address) do - spent = DB.get_inputs(:token, address) - - @unspent_output_index_table - |> :ets.lookup(address) - |> Enum.reduce(spent, fn {_, from, token_address, token_id}, acc -> - [{_, amount, spent?, timestamp, protocol_version}] = - :ets.lookup(@ledger_table, {address, from, token_address, token_id}) - - [ - %VersionedTransactionInput{ - input: %TransactionInput{ - from: from, - amount: amount, - type: {:token, token_address, token_id}, - spent?: spent?, - timestamp: timestamp - }, - protocol_version: protocol_version - } - | acc - ] - end) + unspent_inputs = + @unspent_output_index_table + |> :ets.lookup(address) + |> Enum.reduce([], fn {_, from, token_address, token_id}, acc -> + [{_, amount, spent?, timestamp, protocol_version}] = + :ets.lookup(@ledger_table, {address, from, token_address, token_id}) + + [ + %VersionedTransactionInput{ + input: %TransactionInput{ + from: from, + amount: amount, + type: {:token, token_address, token_id}, + spent?: spent?, + timestamp: timestamp + }, + protocol_version: protocol_version + } + | acc + ] + end) + + # inputs are either all spent or all unspent at a given address + case unspent_inputs do + [] -> + DB.get_inputs(:token, address) + + _ -> + unspent_inputs + end end end diff --git a/lib/archethic/account/mem_tables/uco_ledger.ex b/lib/archethic/account/mem_tables/uco_ledger.ex index f88a57e2f..0843f7eae 100644 --- a/lib/archethic/account/mem_tables/uco_ledger.ex +++ b/lib/archethic/account/mem_tables/uco_ledger.ex @@ -185,7 +185,7 @@ defmodule Archethic.Account.MemTables.UCOLedger do ] end) - # UCO inputs are either all spent or all unspent at a given address + # inputs are either all spent or all unspent at a given address case unspent_inputs do [] -> DB.get_inputs(:UCO, address) From c2c50750c47feacb07f70d164c0e5c7b39d97441 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Mon, 21 Nov 2022 17:10:16 +0100 Subject: [PATCH 16/19] inputs serialization/deserialization improvements --- .../db/embedded_impl/inputs_reader.ex | 20 +++---------------- .../db/embedded_impl/inputs_writer.ex | 6 ++---- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/lib/archethic/db/embedded_impl/inputs_reader.ex b/lib/archethic/db/embedded_impl/inputs_reader.ex index 253b3bf72..f29e20dfa 100644 --- a/lib/archethic/db/embedded_impl/inputs_reader.ex +++ b/lib/archethic/db/embedded_impl/inputs_reader.ex @@ -19,30 +19,16 @@ defmodule Archethic.DB.EmbeddedImpl.InputsReader do {:ok, bin} -> bin |> deserialize_inputs_file([]) - |> Enum.reverse() end end - defp deserialize_inputs_file(<<>>, acc), do: acc + defp deserialize_inputs_file(<<>>, acc), do: Enum.reverse(acc) defp deserialize_inputs_file(bitstring, acc) do {input_bit_size, rest} = Utils.VarInt.get_value(bitstring) + <> = rest - # every serialization contains some padding to be a binary (multipe of 8bits) - {input_bitstring, rest} = - case rem(input_bit_size, 8) do - 0 -> - <> = rest - {input_bitstring, rest} - - remainder -> - <> = rest - - {input_bitstring, rest} - end - - {input, <<>>} = VersionedTransactionInput.deserialize(input_bitstring) + {input, _padding} = VersionedTransactionInput.deserialize(input_bitstring) deserialize_inputs_file(rest, [input | acc]) end end diff --git a/lib/archethic/db/embedded_impl/inputs_writer.ex b/lib/archethic/db/embedded_impl/inputs_writer.ex index c15f4e8fb..3a909b94c 100644 --- a/lib/archethic/db/embedded_impl/inputs_writer.ex +++ b/lib/archethic/db/embedded_impl/inputs_writer.ex @@ -69,11 +69,9 @@ defmodule Archethic.DB.EmbeddedImpl.InputsWriter do def handle_call({:append_input, input}, _from, state = %{fd: fd}) do # we need to pad the bitstring to be a binary # we also need to prefix with the number of bits to be able to ignore padding to deserialize - serialized_input = VersionedTransactionInput.serialize(input) + serialized_input = Utils.wrap_binary(VersionedTransactionInput.serialize(input)) encoded_size = Utils.VarInt.from_value(bit_size(serialized_input)) - wrapped_bin = Utils.wrap_binary(<>) - - IO.binwrite(fd, wrapped_bin) + IO.binwrite(fd, <>) {:reply, :ok, state} end end From 43bc59dbe8499d7194e107cedd271337cea1f618 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Mon, 21 Nov 2022 17:10:52 +0100 Subject: [PATCH 17/19] refactor ledger.get_inputs/1 for readability --- .../account/mem_tables/token_ledger.ex | 29 +++++++------------ .../account/mem_tables/uco_ledger.ex | 29 +++++++------------ 2 files changed, 20 insertions(+), 38 deletions(-) diff --git a/lib/archethic/account/mem_tables/token_ledger.ex b/lib/archethic/account/mem_tables/token_ledger.ex index 99036afef..894f254b3 100644 --- a/lib/archethic/account/mem_tables/token_ledger.ex +++ b/lib/archethic/account/mem_tables/token_ledger.ex @@ -167,14 +167,15 @@ defmodule Archethic.Account.MemTables.TokenLedger do """ @spec get_inputs(binary()) :: list(VersionedTransactionInput.t()) def get_inputs(address) when is_binary(address) do - unspent_inputs = - @unspent_output_index_table - |> :ets.lookup(address) - |> Enum.reduce([], fn {_, from, token_address, token_id}, acc -> - [{_, amount, spent?, timestamp, protocol_version}] = - :ets.lookup(@ledger_table, {address, from, token_address, token_id}) - - [ + case :ets.lookup(@unspent_output_index_table, address) do + [] -> + DB.get_inputs(:token, address) + + inputs -> + Enum.map(inputs, fn {_, from, token_address, token_id} -> + [{_, amount, spent?, timestamp, protocol_version}] = + :ets.lookup(@ledger_table, {address, from, token_address, token_id}) + %VersionedTransactionInput{ input: %TransactionInput{ from: from, @@ -185,17 +186,7 @@ defmodule Archethic.Account.MemTables.TokenLedger do }, protocol_version: protocol_version } - | acc - ] - end) - - # inputs are either all spent or all unspent at a given address - case unspent_inputs do - [] -> - DB.get_inputs(:token, address) - - _ -> - unspent_inputs + end) end end end diff --git a/lib/archethic/account/mem_tables/uco_ledger.ex b/lib/archethic/account/mem_tables/uco_ledger.ex index 0843f7eae..3fa6729b0 100644 --- a/lib/archethic/account/mem_tables/uco_ledger.ex +++ b/lib/archethic/account/mem_tables/uco_ledger.ex @@ -162,14 +162,15 @@ defmodule Archethic.Account.MemTables.UCOLedger do """ @spec get_inputs(binary()) :: list(VersionedTransactionInput.t()) def get_inputs(address) when is_binary(address) do - unspent_inputs = - @unspent_output_index_table - |> :ets.lookup(address) - |> Enum.reduce([], fn {_, from}, acc -> - [{_, amount, spent?, timestamp, reward?, protocol_version}] = - :ets.lookup(@ledger_table, {address, from}) - - [ + case :ets.lookup(@unspent_output_index_table, address) do + [] -> + DB.get_inputs(:UCO, address) + + inputs -> + Enum.map(inputs, fn {_, from} -> + [{_, amount, spent?, timestamp, reward?, protocol_version}] = + :ets.lookup(@ledger_table, {address, from}) + %VersionedTransactionInput{ input: %TransactionInput{ from: from, @@ -181,17 +182,7 @@ defmodule Archethic.Account.MemTables.UCOLedger do }, protocol_version: protocol_version } - | acc - ] - end) - - # inputs are either all spent or all unspent at a given address - case unspent_inputs do - [] -> - DB.get_inputs(:UCO, address) - - _ -> - unspent_inputs + end) end end end From cb2a2e646d74c9e34f48fd9f494ed72c030c3b80 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Tue, 22 Nov 2022 10:10:05 +0100 Subject: [PATCH 18/19] LINT: use protocol_version() instead of hardcoded num --- .../account/mem_tables_loader_test.exs | 179 ++++++++---------- 1 file changed, 81 insertions(+), 98 deletions(-) diff --git a/test/archethic/account/mem_tables_loader_test.exs b/test/archethic/account/mem_tables_loader_test.exs index f57ab82a8..2da3cffd6 100644 --- a/test/archethic/account/mem_tables_loader_test.exs +++ b/test/archethic/account/mem_tables_loader_test.exs @@ -48,7 +48,7 @@ defmodule Archethic.Account.MemTablesLoaderTest do address: "@RewardToken0", type: :mint_rewards, validation_stamp: %ValidationStamp{ - protocol_version: 1, + protocol_version: ArchethicCase.current_protocol_version(), ledger_operations: %LedgerOperations{fee: 0} } }, @@ -56,7 +56,7 @@ defmodule Archethic.Account.MemTablesLoaderTest do address: "@RewardToken1", type: :mint_rewards, validation_stamp: %ValidationStamp{ - protocol_version: 1, + protocol_version: ArchethicCase.current_protocol_version(), ledger_operations: %LedgerOperations{fee: 0} } }, @@ -64,7 +64,7 @@ defmodule Archethic.Account.MemTablesLoaderTest do address: "@RewardToken2", type: :mint_rewards, validation_stamp: %ValidationStamp{ - protocol_version: 1, + protocol_version: ArchethicCase.current_protocol_version(), ledger_operations: %LedgerOperations{fee: 0} } }, @@ -72,7 +72,7 @@ defmodule Archethic.Account.MemTablesLoaderTest do address: "@RewardToken3", type: :mint_rewards, validation_stamp: %ValidationStamp{ - protocol_version: 1, + protocol_version: ArchethicCase.current_protocol_version(), ledger_operations: %LedgerOperations{fee: 0} } }, @@ -80,7 +80,7 @@ defmodule Archethic.Account.MemTablesLoaderTest do address: "@RewardToken4", type: :mint_rewards, validation_stamp: %ValidationStamp{ - protocol_version: 1, + protocol_version: ArchethicCase.current_protocol_version(), ledger_operations: %LedgerOperations{fee: 0} } } @@ -109,48 +109,44 @@ defmodule Archethic.Account.MemTablesLoaderTest do timestamp = DateTime.utc_now() |> DateTime.truncate(:millisecond) assert :ok = MemTablesLoader.load_transaction(create_transaction(timestamp)) - [ - %VersionedUnspentOutput{ - unspent_output: %UnspentOutput{ - from: "@Charlie3", - amount: 1_900_000_000, - type: :UCO, - timestamp: ^timestamp - }, - protocol_version: 1 - }, - %VersionedUnspentOutput{ - unspent_output: %UnspentOutput{ - from: "@Alice2", - amount: 200_000_000, - type: :UCO, - timestamp: ^timestamp - }, - protocol_version: 1 - } - ] = UCOLedger.get_unspent_outputs("@Charlie3") + assert [ + %VersionedUnspentOutput{ + unspent_output: %UnspentOutput{ + from: "@Charlie3", + amount: 1_900_000_000, + type: :UCO, + timestamp: ^timestamp + } + }, + %VersionedUnspentOutput{ + unspent_output: %UnspentOutput{ + from: "@Alice2", + amount: 200_000_000, + type: :UCO, + timestamp: ^timestamp + } + } + ] = UCOLedger.get_unspent_outputs("@Charlie3") - [ - %VersionedUnspentOutput{ - unspent_output: %UnspentOutput{ - from: "@Charlie3", - amount: 3_400_000_000, - timestamp: ^timestamp - }, - protocol_version: 1 - } - ] = UCOLedger.get_unspent_outputs("@Tom4") + assert [ + %VersionedUnspentOutput{ + unspent_output: %UnspentOutput{ + from: "@Charlie3", + amount: 3_400_000_000, + timestamp: ^timestamp + } + } + ] = UCOLedger.get_unspent_outputs("@Tom4") - [ - %VersionedUnspentOutput{ - unspent_output: %UnspentOutput{ - from: "@Charlie3", - amount: 100_000_000, - timestamp: ^timestamp - }, - protocol_version: 1 - } - ] = UCOLedger.get_unspent_outputs(LedgerOperations.burning_address()) + assert [ + %VersionedUnspentOutput{ + unspent_output: %UnspentOutput{ + from: "@Charlie3", + amount: 100_000_000, + timestamp: ^timestamp + } + } + ] = UCOLedger.get_unspent_outputs(LedgerOperations.burning_address()) assert [ %VersionedUnspentOutput{ @@ -159,8 +155,7 @@ defmodule Archethic.Account.MemTablesLoaderTest do amount: 1_000_000_000, type: {:token, "@CharlieToken", 0}, timestamp: ^timestamp - }, - protocol_version: 1 + } } ] = TokenLedger.get_unspent_outputs("@Bob3") end @@ -181,38 +176,35 @@ defmodule Archethic.Account.MemTablesLoaderTest do test "should query DB to load all the transactions", %{timestamp: timestamp} do assert {:ok, _} = MemTablesLoader.start_link() - [ - %VersionedUnspentOutput{ - unspent_output: %UnspentOutput{ - from: "@Charlie3", - amount: 1_900_000_000, - type: :UCO, - timestamp: ^timestamp - }, - protocol_version: 1 - }, - %VersionedUnspentOutput{ - unspent_output: %UnspentOutput{ - from: "@Alice2", - amount: 200_000_000, - type: :UCO, - timestamp: ^timestamp - }, - protocol_version: 1 - } - ] = UCOLedger.get_unspent_outputs("@Charlie3") + assert [ + %VersionedUnspentOutput{ + unspent_output: %UnspentOutput{ + from: "@Charlie3", + amount: 1_900_000_000, + type: :UCO, + timestamp: ^timestamp + } + }, + %VersionedUnspentOutput{ + unspent_output: %UnspentOutput{ + from: "@Alice2", + amount: 200_000_000, + type: :UCO, + timestamp: ^timestamp + } + } + ] = UCOLedger.get_unspent_outputs("@Charlie3") - [ - %VersionedUnspentOutput{ - unspent_output: %UnspentOutput{ - from: "@Charlie3", - amount: 3_400_000_000, - type: :UCO, - timestamp: ^timestamp - }, - protocol_version: 1 - } - ] = UCOLedger.get_unspent_outputs("@Tom4") + assert [ + %VersionedUnspentOutput{ + unspent_output: %UnspentOutput{ + from: "@Charlie3", + amount: 3_400_000_000, + type: :UCO, + timestamp: ^timestamp + } + } + ] = UCOLedger.get_unspent_outputs("@Tom4") assert [ %VersionedUnspentOutput{ @@ -221,8 +213,7 @@ defmodule Archethic.Account.MemTablesLoaderTest do amount: 1_000_000_000, type: {:token, "@CharlieToken", 0}, timestamp: ^timestamp - }, - protocol_version: 1 + } } ] = TokenLedger.get_unspent_outputs("@Bob3") end @@ -233,7 +224,7 @@ defmodule Archethic.Account.MemTablesLoaderTest do address: "@Charlie3", previous_public_key: "Charlie2", validation_stamp: %ValidationStamp{ - protocol_version: 1, + protocol_version: ArchethicCase.current_protocol_version(), timestamp: timestamp, ledger_operations: %LedgerOperations{ fee: 100_000_000, @@ -284,12 +275,10 @@ defmodule Archethic.Account.MemTablesLoaderTest do amount: 1_900_000_000, type: :UCO, timestamp: ^validation_time - }, - protocol_version: 1 + } }, %VersionedUnspentOutput{ - unspent_output: %UnspentOutput{from: "@Alice2", amount: 200_000_000, type: :UCO}, - protocol_version: 1 + unspent_output: %UnspentOutput{from: "@Alice2", amount: 200_000_000, type: :UCO} } ] = UCOLedger.get_unspent_outputs("@Charlie3") @@ -300,8 +289,7 @@ defmodule Archethic.Account.MemTablesLoaderTest do amount: 3_600_000_000, type: :UCO, timestamp: ^validation_time - }, - protocol_version: 1 + } } ] = UCOLedger.get_unspent_outputs("@Tom4") @@ -312,8 +300,7 @@ defmodule Archethic.Account.MemTablesLoaderTest do amount: 200_000_000, type: :UCO, timestamp: ^validation_time - }, - protocol_version: 1 + } } ] = UCOLedger.get_unspent_outputs("@Bob3") @@ -326,8 +313,7 @@ defmodule Archethic.Account.MemTablesLoaderTest do reward?: false, timestamp: ^timestamp, type: {:token, "@WeatherNFT", 1} - }, - protocol_version: 1 + } }, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ @@ -335,8 +321,7 @@ defmodule Archethic.Account.MemTablesLoaderTest do amount: 5_000_000_000, type: {:token, "@RewardToken2", 0}, timestamp: ^validation_time - }, - protocol_version: 1 + } }, %VersionedUnspentOutput{ unspent_output: %UnspentOutput{ @@ -344,8 +329,7 @@ defmodule Archethic.Account.MemTablesLoaderTest do amount: 5_000_000_000, type: {:token, "@RewardToken1", 0}, timestamp: ^validation_time - }, - protocol_version: 1 + } } ] = TokenLedger.get_unspent_outputs("@Charlie3") @@ -358,8 +342,7 @@ defmodule Archethic.Account.MemTablesLoaderTest do amount: 1_000_000_000, type: {:token, "@CharlieToken", 0}, timestamp: ^validation_time - }, - protocol_version: 1 + } } ] = TokenLedger.get_unspent_outputs("@Bob3") end @@ -370,7 +353,7 @@ defmodule Archethic.Account.MemTablesLoaderTest do address: "@Charlie3", previous_public_key: "Charlie2", validation_stamp: %ValidationStamp{ - protocol_version: 1, + protocol_version: ArchethicCase.current_protocol_version(), timestamp: validation_time, ledger_operations: %LedgerOperations{ fee: 0, From 24f5c3082051d1f8a5f7b1ae578fd9d3ef6b32ae Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Tue, 22 Nov 2022 10:20:44 +0100 Subject: [PATCH 19/19] Perf: remove a enum.reverse since we do not care about inputs order --- .../db/embedded_impl/inputs_reader.ex | 2 +- .../db/embedded_impl/inputs_test.exs | 70 ++++++++++++++++++- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/lib/archethic/db/embedded_impl/inputs_reader.ex b/lib/archethic/db/embedded_impl/inputs_reader.ex index f29e20dfa..12f269a20 100644 --- a/lib/archethic/db/embedded_impl/inputs_reader.ex +++ b/lib/archethic/db/embedded_impl/inputs_reader.ex @@ -22,7 +22,7 @@ defmodule Archethic.DB.EmbeddedImpl.InputsReader do end end - defp deserialize_inputs_file(<<>>, acc), do: Enum.reverse(acc) + defp deserialize_inputs_file(<<>>, acc), do: acc defp deserialize_inputs_file(bitstring, acc) do {input_bit_size, rest} = Utils.VarInt.get_value(bitstring) diff --git a/test/archethic/db/embedded_impl/inputs_test.exs b/test/archethic/db/embedded_impl/inputs_test.exs index 9448cded5..358600c72 100644 --- a/test/archethic/db/embedded_impl/inputs_test.exs +++ b/test/archethic/db/embedded_impl/inputs_test.exs @@ -77,7 +77,40 @@ defmodule Archethic.DB.EmbeddedImpl.InputsTest do {:ok, pid1} = InputsWriter.start_link(:UCO, address) Enum.each(inputs, &InputsWriter.append_input(pid1, &1)) - assert ^inputs = InputsReader.get_inputs(:UCO, address) + result_inputs = InputsReader.get_inputs(:UCO, address) + + assert Enum.any?( + result_inputs, + &(&1 == + %VersionedTransactionInput{ + protocol_version: 1, + input: %TransactionInput{ + amount: 1, + type: :UCO, + from: address2, + reward?: true, + spent?: true, + timestamp: ~U[2022-11-14 14:54:12Z] + } + }) + ) + + assert Enum.any?( + result_inputs, + &(&1 == + %VersionedTransactionInput{ + protocol_version: 1, + input: %TransactionInput{ + amount: 2, + type: :UCO, + from: address3, + reward?: true, + spent?: true, + timestamp: ~U[2022-11-14 14:54:12Z] + } + }) + ) + assert [] = InputsReader.get_inputs(:UCO, address2) assert [] = InputsReader.get_inputs(:UCO, address3) end @@ -116,7 +149,40 @@ defmodule Archethic.DB.EmbeddedImpl.InputsTest do {:ok, pid1} = InputsWriter.start_link(:token, address) Enum.each(inputs, &InputsWriter.append_input(pid1, &1)) - assert ^inputs = InputsReader.get_inputs(:token, address) + result_inputs = InputsReader.get_inputs(:token, address) + + assert Enum.any?( + result_inputs, + &(&1 == + %VersionedTransactionInput{ + protocol_version: 1, + input: %TransactionInput{ + amount: 1, + type: {:token, token_address, 0}, + from: address2, + reward?: true, + spent?: true, + timestamp: ~U[2022-11-14 14:54:12Z] + } + }) + ) + + assert Enum.any?( + result_inputs, + &(&1 == + %VersionedTransactionInput{ + protocol_version: 1, + input: %TransactionInput{ + amount: 2, + type: {:token, token_address, 0}, + from: address3, + reward?: true, + spent?: true, + timestamp: ~U[2022-11-14 14:54:12Z] + } + }) + ) + assert [] = InputsReader.get_inputs(:token, address2) assert [] = InputsReader.get_inputs(:token, address3) end