diff --git a/lib/archethic.ex b/lib/archethic.ex index 6ece12b7b..bac0f5f8d 100644 --- a/lib/archethic.ex +++ b/lib/archethic.ex @@ -11,8 +11,6 @@ defmodule Archethic do alias __MODULE__.P2P alias __MODULE__.P2P.Node - alias __MODULE__.DB - alias __MODULE__.P2P.Message.Balance alias __MODULE__.P2P.Message.GetBalance alias __MODULE__.P2P.Message.NewTransaction @@ -275,7 +273,7 @@ defmodule Archethic do try do {local_chain, paging_address} = with true <- paging_address != nil, - true <- DB.transaction_exists?(paging_address), + true <- TransactionChain.transaction_exists?(paging_address), last_address when last_address != nil <- TransactionChain.get_last_local_address(address), true <- last_address != paging_address do diff --git a/lib/archethic/account.ex b/lib/archethic/account.ex index 780a9e27d..b1b4690a6 100644 --- a/lib/archethic/account.ex +++ b/lib/archethic/account.ex @@ -60,6 +60,6 @@ defmodule Archethic.Account do @doc """ Load the transaction into the Account context filling the memory tables for ledgers """ - @spec load_transaction(Transaction.t()) :: :ok - defdelegate load_transaction(transaction), to: MemTablesLoader + @spec load_transaction(Transaction.t(), boolean()) :: :ok + defdelegate load_transaction(transaction, io_transaction?), to: MemTablesLoader end diff --git a/lib/archethic/account/mem_tables_loader.ex b/lib/archethic/account/mem_tables_loader.ex index 69e31e1e9..282da484d 100644 --- a/lib/archethic/account/mem_tables_loader.ex +++ b/lib/archethic/account/mem_tables_loader.ex @@ -56,9 +56,13 @@ defmodule Archethic.Account.MemTablesLoader do @spec init(args :: list()) :: {:ok, []} def init(_args) do + TransactionChain.list_io_transactions(@query_fields) + |> Stream.each(&load_transaction(&1, true)) + |> Stream.run() + TransactionChain.list_all(@query_fields) |> Stream.reject(&(&1.type in @excluded_types)) - |> Stream.each(&load_transaction/1) + |> Stream.each(&load_transaction(&1, false)) |> Stream.run() {:ok, []} @@ -67,25 +71,30 @@ defmodule Archethic.Account.MemTablesLoader do @doc """ Load the transaction into the memory tables """ - @spec load_transaction(Transaction.t()) :: :ok - def load_transaction(%Transaction{ - address: address, - type: tx_type, - previous_public_key: previous_public_key, - validation_stamp: %ValidationStamp{ - timestamp: timestamp, - protocol_version: protocol_version, - ledger_operations: %LedgerOperations{ - fee: fee, - unspent_outputs: unspent_outputs, - transaction_movements: transaction_movements + @spec load_transaction(Transaction.t(), boolean()) :: :ok + def load_transaction( + %Transaction{ + address: address, + type: tx_type, + previous_public_key: previous_public_key, + validation_stamp: %ValidationStamp{ + timestamp: timestamp, + protocol_version: protocol_version, + ledger_operations: %LedgerOperations{ + fee: fee, + unspent_outputs: unspent_outputs, + transaction_movements: transaction_movements + } } - } - }) do - previous_address = Crypto.derive_address(previous_public_key) - - UCOLedger.spend_all_unspent_outputs(previous_address) - TokenLedger.spend_all_unspent_outputs(previous_address) + }, + io_transaction? + ) do + unless io_transaction? do + previous_address = Crypto.derive_address(previous_public_key) + + UCOLedger.spend_all_unspent_outputs(previous_address) + TokenLedger.spend_all_unspent_outputs(previous_address) + end burn_storage_nodes = Election.storage_nodes( diff --git a/lib/archethic/contracts/loader.ex b/lib/archethic/contracts/loader.ex index b4686a375..29247d095 100644 --- a/lib/archethic/contracts/loader.ex +++ b/lib/archethic/contracts/loader.ex @@ -27,17 +27,19 @@ defmodule Archethic.Contracts.Loader do def init(_opts) do DB.list_last_transaction_addresses() |> Stream.map(fn address -> - {:ok, tx} = - DB.get_transaction(address, [ - :address, - :previous_public_key, - :data, - validation_stamp: [:timestamp] - ]) - - tx + DB.get_transaction(address, [ + :address, + :previous_public_key, + :data, + validation_stamp: [:timestamp] + ]) end) - |> Stream.filter(&(&1.data.code != "")) + |> Stream.filter(fn + {:ok, %Transaction{data: %TransactionData{code: ""}}} -> false + {:error, _} -> false + _ -> true + end) + |> Stream.map(fn {:ok, tx} -> tx end) |> Stream.each(&load_transaction(&1, true)) |> Stream.run() diff --git a/lib/archethic/db.ex b/lib/archethic/db.ex index b638bd9aa..ee5efd4e1 100644 --- a/lib/archethic/db.ex +++ b/lib/archethic/db.ex @@ -13,6 +13,8 @@ defmodule Archethic.DB do use Knigge, otp_app: :archethic, default: EmbeddedImpl + @type storage_type() :: :chain | :io + @callback get_transaction(address :: binary(), fields :: list()) :: {:ok, Transaction.t()} | {:error, :transaction_not_exists} @callback get_beacon_summary(summary_address :: binary()) :: @@ -24,12 +26,13 @@ defmodule Archethic.DB do fields :: list(), opts :: [paging_state: nil | binary(), after: DateTime.t()] ) :: Enumerable.t() - @callback write_transaction(Transaction.t()) :: :ok + @callback write_transaction(Transaction.t(), storage_type()) :: :ok @callback write_beacon_summary(Summary.t()) :: :ok @callback clear_beacon_summaries() :: :ok @callback write_beacon_summaries_aggregate(SummaryAggregate.t()) :: :ok @callback write_transaction_chain(Enumerable.t()) :: :ok @callback list_transactions(fields :: list()) :: Enumerable.t() + @callback list_io_transactions(fields :: list()) :: Enumerable.t() @callback add_last_transaction_address(binary(), binary(), DateTime.t()) :: :ok @callback list_last_transaction_addresses() :: Enumerable.t() @@ -61,7 +64,7 @@ defmodule Archethic.DB do @callback get_latest_burned_fees() :: non_neg_integer() @callback get_nb_transactions() :: non_neg_integer() - @callback transaction_exists?(binary()) :: boolean() + @callback transaction_exists?(binary(), storage_type()) :: boolean() @callback register_p2p_summary(list({Crypto.key(), boolean(), float(), DateTime.t()})) :: :ok diff --git a/lib/archethic/db/embedded_impl.ex b/lib/archethic/db/embedded_impl.ex index 0136e7838..91800661c 100644 --- a/lib/archethic/db/embedded_impl.ex +++ b/lib/archethic/db/embedded_impl.ex @@ -9,6 +9,8 @@ defmodule Archethic.DB.EmbeddedImpl do alias Archethic.Crypto + alias Archethic.DB + alias __MODULE__.BootstrapInfo alias __MODULE__.ChainIndex alias __MODULE__.ChainReader @@ -74,6 +76,9 @@ defmodule Archethic.DB.EmbeddedImpl do Enum.each(sorted_chain, fn tx -> unless ChainIndex.transaction_exists?(tx.address, db_path()) do ChainWriter.append_transaction(genesis_address, tx) + + # Delete IO transaction if it exists as it is now stored as a chain + delete_io_transaction(tx.address) end end) end @@ -81,31 +86,44 @@ defmodule Archethic.DB.EmbeddedImpl do @doc """ Write a single transaction and append it to its chain """ - @spec write_transaction(Transaction.t()) :: :ok - def write_transaction(tx = %Transaction{}) do + @spec write_transaction(Transaction.t(), DB.storage_type()) :: :ok + def write_transaction(tx, storage_type \\ :chain) + + def write_transaction(tx = %Transaction{}, :chain) do if ChainIndex.transaction_exists?(tx.address, db_path()) do {:error, :transaction_already_exists} else previous_address = Transaction.previous_address(tx) - case ChainIndex.get_tx_entry(previous_address, db_path()) do - {:ok, %{genesis_address: genesis_address}} -> - do_write_transaction(genesis_address, tx) + genesis_address = + case ChainIndex.get_tx_entry(previous_address, db_path()) do + {:ok, %{genesis_address: genesis_address}} -> + genesis_address - {:error, :not_exists} -> - ChainWriter.append_transaction(previous_address, tx) - end + {:error, :not_exists} -> + previous_address + end + + ChainWriter.append_transaction(genesis_address, tx) + + # Delete IO transaction if it exists as it is now stored as a chain + delete_io_transaction(tx.address) end end - defp do_write_transaction(genesis_address, tx) do - if ChainIndex.transaction_exists?(tx.address, db_path()) do + def write_transaction(tx = %Transaction{}, :io) do + if ChainIndex.transaction_exists?(tx.address, :io, db_path()) do {:error, :transaction_already_exists} else - ChainWriter.append_transaction(genesis_address, tx) + ChainWriter.write_io_transaction(tx, db_path()) end end + defp delete_io_transaction(address) do + ChainWriter.io_path(db_path(), address) |> File.rm() + :ok + end + @doc """ Write a beacon summary in DB """ @@ -137,9 +155,9 @@ defmodule Archethic.DB.EmbeddedImpl do @doc """ Determine if the transaction exists or not """ - @spec transaction_exists?(address :: binary()) :: boolean() - def transaction_exists?(address) when is_binary(address) do - ChainIndex.transaction_exists?(address, db_path()) + @spec transaction_exists?(address :: binary(), storage_type :: DB.storage_type()) :: boolean() + def transaction_exists?(address, storage_type) when is_binary(address) do + ChainIndex.transaction_exists?(address, storage_type, db_path()) end @doc """ @@ -278,7 +296,7 @@ defmodule Archethic.DB.EmbeddedImpl do end @doc """ - List all the transactions + List all the transactions in chain storage """ @spec list_transactions(fields :: list()) :: Enumerable.t() | list(Transaction.t()) def list_transactions(fields \\ []) when is_list(fields) do @@ -286,6 +304,14 @@ defmodule Archethic.DB.EmbeddedImpl do |> Stream.flat_map(&ChainReader.stream_scan_chain(&1, nil, fields, db_path())) end + @doc """ + List all the transactions in io storage + """ + @spec list_io_transactions(fields :: list()) :: Enumerable.t() | list(Transaction.t()) + def list_io_transactions(fields \\ []) do + ChainReader.list_io_transactions(fields, db_path()) + end + @doc """ List all the last transaction chain addresses """ diff --git a/lib/archethic/db/embedded_impl/chain_index.ex b/lib/archethic/db/embedded_impl/chain_index.ex index 1da38d4d8..b5f05e8e1 100644 --- a/lib/archethic/db/embedded_impl/chain_index.ex +++ b/lib/archethic/db/embedded_impl/chain_index.ex @@ -7,6 +7,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do @vsn Mix.Project.config()[:version] alias Archethic.Crypto + alias Archethic.DB alias Archethic.DB.EmbeddedImpl.ChainWriter alias Archethic.TransactionChain.Transaction @@ -196,14 +197,18 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do @doc """ Determine if a transaction exists """ - @spec transaction_exists?(binary(), String.t()) :: boolean() - def transaction_exists?(address = <<_::8, _::8, _subset::8, _digest::binary>>, db_path) do + @spec transaction_exists?(binary(), DB.storage_type(), String.t()) :: boolean() + def transaction_exists?(address, storage_type \\ :chain, db_path) + + def transaction_exists?(address, storage_type, db_path) do case get_tx_entry(address, db_path) do {:ok, _} -> true {:error, :not_exists} -> - false + if storage_type == :io, + do: ChainWriter.io_path(db_path, address) |> File.exists?(), + else: false end end diff --git a/lib/archethic/db/embedded_impl/chain_reader.ex b/lib/archethic/db/embedded_impl/chain_reader.ex index 57cca625f..ebffc3e55 100644 --- a/lib/archethic/db/embedded_impl/chain_reader.ex +++ b/lib/archethic/db/embedded_impl/chain_reader.ex @@ -139,6 +139,60 @@ defmodule Archethic.DB.EmbeddedImpl.ChainReader do end end + @doc """ + List all the transactions in io storage + """ + @spec list_io_transactions(fields :: list(), db_path :: String.t()) :: + Enumerable.t() | list(Transaction.t()) + def list_io_transactions(fields, db_path) do + io_transactions_path = + ChainWriter.base_io_path(db_path) + |> Path.join("*") + |> Path.wildcard() + + Stream.resource( + fn -> io_transactions_path end, + fn + [filepath | rest] -> {[read_io_transaction(filepath, fields)], rest} + [] -> {:halt, nil} + end, + fn _ -> :ok end + ) + end + + defp read_io_transaction(filepath, fields) do + # Open the file as the position from the transaction in the chain file + fd = File.open!(filepath, [:binary, :read]) + + {:ok, <>} = :file.read(fd, 8) + column_names = fields_to_column_names(fields) + + # Ensure the validation stamp's protocol version is retrieved if we fetch validation stamp fields + has_validation_stamp_fields? = + Enum.any?(column_names, &String.starts_with?(&1, "validation_stamp.")) + + has_validation_stamp_protocol_field? = + Enum.any?(column_names, &(&1 == "validation_stamp.protocol_version")) + + column_names = + if has_validation_stamp_fields? and !has_validation_stamp_protocol_field? do + ["validation_stamp.protocol_version" | column_names] + else + column_names + end + + # Read the transaction and extract requested columns from the fields arg + tx = + fd + |> read_transaction(column_names, size, 0) + |> Enum.into(%{}) + |> decode_transaction_columns(version) + + :file.close(fd) + + tx + end + defp process_get_chain(fd, address, fields, opts, db_path) do # Set the file cursor position to the paging state case Keyword.get(opts, :paging_state) do diff --git a/lib/archethic/db/embedded_impl/chain_writer.ex b/lib/archethic/db/embedded_impl/chain_writer.ex index f74654888..8ee44d522 100644 --- a/lib/archethic/db/embedded_impl/chain_writer.ex +++ b/lib/archethic/db/embedded_impl/chain_writer.ex @@ -31,6 +31,28 @@ defmodule Archethic.DB.EmbeddedImpl.ChainWriter do GenServer.call(pid, {:append_tx, genesis_address, tx}) end + @doc """ + write an io transaction in a file name by it's address + """ + @spec write_io_transaction(Transaction.t(), String.t()) :: :ok + def write_io_transaction(tx = %Transaction{address: address}, db_path) do + start = System.monotonic_time() + + filename = io_path(db_path, address) + + data = Encoding.encode(tx) + + File.write!( + filename, + data, + [:exclusive, :binary] + ) + + :telemetry.execute([:archethic, :db], %{duration: System.monotonic_time() - start}, %{ + query: "write_io_transaction" + }) + end + @doc """ Write a beacon summary in a new file """ @@ -99,6 +121,10 @@ defmodule Archethic.DB.EmbeddedImpl.ChainWriter do |> base_chain_path() |> File.mkdir_p!() + path + |> base_io_path() + |> File.mkdir_p!() + path |> base_beacon_path() |> File.mkdir_p!() @@ -117,6 +143,15 @@ defmodule Archethic.DB.EmbeddedImpl.ChainWriter do {:reply, :ok, state} end + def handle_call( + {:write_io_transaction, tx}, + _from, + state = %{db_path: db_path} + ) do + write_io_transaction(tx, db_path) + {:reply, :ok, state} + end + def terminate(_reason, _state = %{partition: partition}) do :ets.delete(:archethic_db_chain_writers, partition) :ignore @@ -174,6 +209,15 @@ defmodule Archethic.DB.EmbeddedImpl.ChainWriter do Path.join([base_chain_path(db_path), Base.encode16(genesis_address)]) end + @doc """ + Return the path of the io storage location + """ + @spec io_path(String.t(), binary()) :: String.t() + def io_path(db_path, address) + when is_binary(address) and is_binary(db_path) do + Path.join([base_io_path(db_path), Base.encode16(address)]) + end + @doc """ Return the chain base path """ @@ -182,6 +226,14 @@ defmodule Archethic.DB.EmbeddedImpl.ChainWriter do Path.join([db_path, "chains"]) end + @doc """ + Return the io base path + """ + @spec base_io_path(String.t()) :: String.t() + def base_io_path(db_path) do + Path.join([db_path, "io"]) + end + @doc """ Return the path of the beacon summary storage location """ diff --git a/lib/archethic/oracle_chain/scheduler.ex b/lib/archethic/oracle_chain/scheduler.ex index 2985eb1a3..23b35794e 100644 --- a/lib/archethic/oracle_chain/scheduler.ex +++ b/lib/archethic/oracle_chain/scheduler.ex @@ -4,7 +4,7 @@ defmodule Archethic.OracleChain.Scheduler do """ alias Archethic.Crypto - alias Archethic.DB + alias Archethic.Election alias Archethic.P2P @@ -307,7 +307,7 @@ defmodule Archethic.OracleChain.Scheduler do tx = build_oracle_transaction(summary_date, index, new_oracle_data) with {:empty, false} <- {:empty, Enum.empty?(new_oracle_data)}, - {:exists, false} <- {:exists, DB.transaction_exists?(tx.address)}, + {:exists, false} <- {:exists, TransactionChain.transaction_exists?(tx.address)}, {:trigger, true} <- {:trigger, trigger_node?(storage_nodes)} do send_polling_transaction(tx) :keep_state_and_data @@ -364,7 +364,7 @@ defmodule Archethic.OracleChain.Scheduler do storage_nodes = tx_address |> Election.storage_nodes(authorized_nodes) watcher_pid = - with {:exists, false} <- {:exists, DB.transaction_exists?(tx_address)}, + with {:exists, false} <- {:exists, TransactionChain.transaction_exists?(tx_address)}, {:trigger, true} <- {:trigger, trigger_node?(storage_nodes)} do Logger.debug("Oracle transaction summary sending", transaction_address: Base.encode16(tx_address), @@ -699,7 +699,7 @@ defmodule Archethic.OracleChain.Scheduler do next_pub ) - if DB.transaction_exists?(tx.address) do + if TransactionChain.transaction_exists?(tx.address) do Logger.debug( "Transaction Already Exists:oracle summary transaction - aggregation: #{inspect(aggregated_content)}", transaction_address: Base.encode16(tx.address), diff --git a/lib/archethic/replication.ex b/lib/archethic/replication.ex index 8ec0bd010..cbf80d1d3 100644 --- a/lib/archethic/replication.ex +++ b/lib/archethic/replication.ex @@ -111,7 +111,7 @@ defmodule Archethic.Replication do TransactionChain.write_transaction(tx) - :ok = ingest_transaction(tx) + :ok = ingest_transaction(tx, false) Logger.info("Replication finished", transaction_address: Base.encode16(address), @@ -158,7 +158,7 @@ defmodule Archethic.Replication do }, self_repair? \\ false ) do - if TransactionChain.transaction_exists?(address) do + if TransactionChain.transaction_exists?(address, :io) do Logger.warning("Transaction already exists", transaction_address: Base.encode16(address), transaction_type: type @@ -175,8 +175,8 @@ defmodule Archethic.Replication do case TransactionValidator.validate(tx, self_repair?) do :ok -> - :ok = TransactionChain.write_transaction(tx) - ingest_transaction(tx) + :ok = TransactionChain.write_transaction(tx, :io) + ingest_transaction(tx, true) Logger.info("Replication finished", transaction_address: Base.encode16(address), @@ -524,13 +524,13 @@ defmodule Archethic.Replication do - Transactions with smart contract deploy instances of them or can put in pending state waiting approval signatures - Code approval transactions may trigger the TestNets deployments or hot-reloads """ - @spec ingest_transaction(Transaction.t()) :: :ok - def ingest_transaction(tx = %Transaction{}) do + @spec ingest_transaction(Transaction.t(), boolean()) :: :ok + def ingest_transaction(tx = %Transaction{}, io_transaction?) do TransactionChain.load_transaction(tx) Crypto.load_transaction(tx) P2P.load_transaction(tx) SharedSecrets.load_transaction(tx) - Account.load_transaction(tx) + Account.load_transaction(tx, io_transaction?) Contracts.load_transaction(tx) OracleChain.load_transaction(tx) Reward.load_transaction(tx) diff --git a/lib/archethic/self_repair/sync.ex b/lib/archethic/self_repair/sync.ex index bdf8738a7..5ddca9a18 100644 --- a/lib/archethic/self_repair/sync.ex +++ b/lib/archethic/self_repair/sync.ex @@ -232,7 +232,7 @@ defmodule Archethic.SelfRepair.Sync do transactions_to_sync = transaction_summaries - |> Enum.reject(&TransactionChain.transaction_exists?(&1.address)) + |> Enum.reject(&TransactionChain.transaction_exists?(&1.address, :io)) |> Enum.filter(&TransactionHandler.download_transaction?/1) synchronize_transactions(transactions_to_sync, download_nodes) diff --git a/lib/archethic/transaction_chain.ex b/lib/archethic/transaction_chain.ex index ee6ec16b6..3bbd9875f 100644 --- a/lib/archethic/transaction_chain.ex +++ b/lib/archethic/transaction_chain.ex @@ -58,6 +58,12 @@ defmodule Archethic.TransactionChain do @spec list_all(fields :: list()) :: Enumerable.t() defdelegate list_all(fields \\ []), to: DB, as: :list_transactions + @doc """ + List all the io transactions stored + """ + @spec list_io_transactions(fields :: list()) :: Enumerable.t() + defdelegate list_io_transactions(fields \\ []), to: DB + @doc """ List all the transaction for a given transaction type sorted by timestamp in descent order """ @@ -172,14 +178,15 @@ defmodule Archethic.TransactionChain do @doc """ Persist only one transaction """ - @spec write_transaction(Transaction.t()) :: :ok + @spec write_transaction(Transaction.t(), DB.storage_type()) :: :ok def write_transaction( tx = %Transaction{ address: address, type: type - } + }, + storage_type \\ :chain ) do - DB.write_transaction(tx) + DB.write_transaction(tx, storage_type) KOLedger.remove_transaction(address) Logger.info("Transaction stored", @@ -247,8 +254,8 @@ defmodule Archethic.TransactionChain do @doc """ Determine if the transaction exists """ - @spec transaction_exists?(binary()) :: boolean() - defdelegate transaction_exists?(address), to: DB + @spec transaction_exists?(binary(), DB.storage_type()) :: boolean() + defdelegate transaction_exists?(address, storage_type \\ :chain), to: DB @doc """ Return the size of transaction chain diff --git a/lib/archethic/utils/detect_node_responsiveness.ex b/lib/archethic/utils/detect_node_responsiveness.ex index a4f8d0c23..9810bc99b 100644 --- a/lib/archethic/utils/detect_node_responsiveness.ex +++ b/lib/archethic/utils/detect_node_responsiveness.ex @@ -4,9 +4,11 @@ defmodule Archethic.Utils.DetectNodeResponsiveness do """ @default_timeout Application.compile_env(:archethic, __MODULE__, []) |> Keyword.get(:timeout, 5_000) - alias Archethic.DB + alias Archethic.Mining + alias Archethic.TransactionChain + use GenServer @vsn Mix.Project.config()[:version] require Logger @@ -43,7 +45,7 @@ defmodule Archethic.Utils.DetectNodeResponsiveness do timeout: timeout } ) do - with {:exists, false} <- {:exists, DB.transaction_exists?(address)}, + with {:exists, false} <- {:exists, TransactionChain.transaction_exists?(address)}, {:mining, false} <- {:mining, Mining.processing?(address)}, {:remaining, true} <- {:remaining, count < max_retry} do Logger.info("calling replay fn with count=#{count}", diff --git a/test/archethic/account/mem_tables_loader_test.exs b/test/archethic/account/mem_tables_loader_test.exs index 2da3cffd6..27291f2fb 100644 --- a/test/archethic/account/mem_tables_loader_test.exs +++ b/test/archethic/account/mem_tables_loader_test.exs @@ -107,7 +107,9 @@ defmodule Archethic.Account.MemTablesLoaderTest do }) timestamp = DateTime.utc_now() |> DateTime.truncate(:millisecond) - assert :ok = MemTablesLoader.load_transaction(create_transaction(timestamp)) + + assert :ok = + MemTablesLoader.load_transaction(create_transaction(timestamp, "@Charlie3"), false) assert [ %VersionedUnspentOutput{ @@ -166,8 +168,11 @@ defmodule Archethic.Account.MemTablesLoaderTest do timestamp = DateTime.utc_now() |> DateTime.truncate(:millisecond) MockDB + |> stub(:list_io_transactions, fn _fields -> + [create_transaction(timestamp, "@Charlie4")] + end) |> stub(:list_transactions, fn _fields -> - [create_transaction(timestamp)] + [create_transaction(timestamp, "@Charlie3")] end) %{timestamp: timestamp} @@ -203,6 +208,14 @@ defmodule Archethic.Account.MemTablesLoaderTest do type: :UCO, timestamp: ^timestamp } + }, + %VersionedUnspentOutput{ + unspent_output: %UnspentOutput{ + from: "@Charlie4", + amount: 3_400_000_000, + type: :UCO, + timestamp: ^timestamp + } } ] = UCOLedger.get_unspent_outputs("@Tom4") @@ -214,14 +227,22 @@ defmodule Archethic.Account.MemTablesLoaderTest do type: {:token, "@CharlieToken", 0}, timestamp: ^timestamp } + }, + %VersionedUnspentOutput{ + unspent_output: %UnspentOutput{ + from: "@Charlie4", + amount: 1_000_000_000, + type: {:token, "@CharlieToken", 0}, + timestamp: ^timestamp + } } ] = TokenLedger.get_unspent_outputs("@Bob3") end end - defp create_transaction(timestamp) do + defp create_transaction(timestamp, address) do %Transaction{ - address: "@Charlie3", + address: address, previous_public_key: "Charlie2", validation_stamp: %ValidationStamp{ protocol_version: ArchethicCase.current_protocol_version(), @@ -263,9 +284,8 @@ defmodule Archethic.Account.MemTablesLoaderTest do DateTime.utc_now() |> DateTime.add(-86_400) |> DateTime.truncate(:millisecond) assert :ok = - MemTablesLoader.load_transaction( - create_reward_transaction(timestamp, validation_time) - ) + create_reward_transaction(timestamp, validation_time) + |> MemTablesLoader.load_transaction(false) # uco ledger assert [ diff --git a/test/archethic/bootstrap/network_init_test.exs b/test/archethic/bootstrap/network_init_test.exs index fab68996e..45bd774a9 100644 --- a/test/archethic/bootstrap/network_init_test.exs +++ b/test/archethic/bootstrap/network_init_test.exs @@ -228,7 +228,7 @@ defmodule Archethic.Bootstrap.NetworkInitTest do me = self() MockDB - |> stub(:write_transaction, fn ^tx -> + |> stub(:write_transaction, fn ^tx, _ -> send(me, :write_transaction) :ok end) @@ -277,7 +277,7 @@ defmodule Archethic.Bootstrap.NetworkInitTest do me = self() MockDB - |> expect(:write_transaction, fn tx -> + |> expect(:write_transaction, fn tx, _ -> send(me, {:transaction, tx}) :ok end) @@ -429,7 +429,7 @@ defmodule Archethic.Bootstrap.NetworkInitTest do me = self() MockDB - |> expect(:write_transaction, fn tx -> + |> expect(:write_transaction, fn tx, _ -> send(me, {:transaction, tx}) :ok end) diff --git a/test/archethic/bootstrap_test.exs b/test/archethic/bootstrap_test.exs index 0a3c1ac58..5c6f6067a 100644 --- a/test/archethic/bootstrap_test.exs +++ b/test/archethic/bootstrap_test.exs @@ -263,7 +263,7 @@ defmodule Archethic.BootstrapTest do validated_tx = %{tx | validation_stamp: stamp} :ok = TransactionChain.write([validated_tx]) - :ok = Replication.ingest_transaction(validated_tx) + :ok = Replication.ingest_transaction(validated_tx, false) {:ok, %Ok{}} @@ -549,9 +549,9 @@ defmodule Archethic.BootstrapTest do ^addr0 -> {addr2, DateTime.utc_now()} end) |> stub(:transaction_exists?, fn - ^addr4 -> false - ^addr3 -> false - ^addr2 -> true + ^addr4, _ -> false + ^addr3, _ -> false + ^addr2, _ -> true end) |> expect(:get_transaction, fn ^addr3, _ -> {:error, :transaction_not_exists} @@ -562,7 +562,7 @@ defmodule Archethic.BootstrapTest do send(me, {:write_transaction_chain, txn_list}) :ok end) - |> expect(:write_transaction, fn _tx -> + |> expect(:write_transaction, fn _tx, _ -> :ok end) diff --git a/test/archethic/db/embedded_impl_test.exs b/test/archethic/db/embedded_impl_test.exs index 271f8deba..10e621cee 100644 --- a/test/archethic/db/embedded_impl_test.exs +++ b/test/archethic/db/embedded_impl_test.exs @@ -122,18 +122,71 @@ defmodule Archethic.DB.EmbeddedTest do genesis_address: ^genesis_address }} = ChainIndex.get_tx_entry(tx2.address, db_path) end + + test "should write transaction in io storage", %{db_path: db_path} do + tx1 = TransactionFactory.create_valid_transaction() + :ok = EmbeddedImpl.write_transaction(tx1, :io) + + filename = ChainWriter.io_path(db_path, tx1.address) + + assert File.exists?(filename) + + contents = File.read!(filename) + + assert contents == Encoding.encode(tx1) + end + + test "should delete transaction in io storage after writing it in chain storage", %{ + db_path: db_path + } do + tx1 = TransactionFactory.create_valid_transaction() + :ok = EmbeddedImpl.write_transaction(tx1, :io) + + filename_io = ChainWriter.io_path(db_path, tx1.address) + + assert File.exists?(filename_io) + + :ok = EmbeddedImpl.write_transaction(tx1) + + genesis_address = Transaction.previous_address(tx1) + filename_chain = ChainWriter.chain_path(db_path, genesis_address) + + assert File.exists?(filename_chain) + assert !File.exists?(filename_io) + end end - describe "transaction_exists?/1" do - test "should return true when the transaction is present" do + describe "transaction_exists?/2" do + test "should return true when the transaction is present in chain storage" do tx1 = TransactionFactory.create_valid_transaction() :ok = EmbeddedImpl.write_transaction_chain([tx1]) - assert EmbeddedImpl.transaction_exists?(tx1.address) + assert EmbeddedImpl.transaction_exists?(tx1.address, :chain) end test "should return false when the transaction is present" do - assert !EmbeddedImpl.transaction_exists?(:crypto.strong_rand_bytes(32)) + assert !EmbeddedImpl.transaction_exists?(:crypto.strong_rand_bytes(32), :chain) + end + + test "should return false when the transaction is not present in chain storage but in io storage" do + tx1 = TransactionFactory.create_valid_transaction() + :ok = EmbeddedImpl.write_transaction(tx1, :io) + + assert !EmbeddedImpl.transaction_exists?(tx1.address, :chain) + end + + test "should return true when the transaction is present in io storage" do + tx1 = TransactionFactory.create_valid_transaction() + :ok = EmbeddedImpl.write_transaction(tx1, :io) + + assert EmbeddedImpl.transaction_exists?(tx1.address, :io) + end + + test "should return true when the transaction is present in chain storage and asking for io storage" do + tx1 = TransactionFactory.create_valid_transaction() + :ok = EmbeddedImpl.write_transaction(tx1) + + assert EmbeddedImpl.transaction_exists?(tx1.address, :io) end end diff --git a/test/archethic/replication_test.exs b/test/archethic/replication_test.exs index fa95916df..6695310b0 100644 --- a/test/archethic/replication_test.exs +++ b/test/archethic/replication_test.exs @@ -83,7 +83,7 @@ defmodule Archethic.ReplicationTest do # send(me, :replicated) # :ok # end) - |> expect(:write_transaction, fn ^tx -> + |> expect(:write_transaction, fn ^tx, _ -> send(me, :replicated) :ok end) @@ -145,7 +145,7 @@ defmodule Archethic.ReplicationTest do tx = create_valid_transaction(unspent_outputs) MockDB - |> expect(:write_transaction, fn _ -> + |> expect(:write_transaction, fn _, _ -> send(me, :replicated) :ok end) diff --git a/test/archethic/reward_test.exs b/test/archethic/reward_test.exs index 255977e1d..a542a86f2 100644 --- a/test/archethic/reward_test.exs +++ b/test/archethic/reward_test.exs @@ -138,7 +138,7 @@ defmodule Archethic.RewardTest do end test "Balance Should be updated with UCO ,for reward movements " do - Enum.each(get_reward_transactions(), &AccountTablesLoader.load_transaction(&1)) + Enum.each(get_reward_transactions(), &AccountTablesLoader.load_transaction(&1, false)) # @Ada1 # from alen2: 2uco, dan2: 19uco, rewardtoken1: 50, rewardtoken2: 50 @@ -365,7 +365,7 @@ defmodule Archethic.RewardTest do end test "Node Rewards Should not be minted" do - AccountTablesLoader.load_transaction(get_node_reward_txns()) + AccountTablesLoader.load_transaction(get_node_reward_txns(), false) assert %{ uco: 0, diff --git a/test/archethic/self_repair/repair_worker_test.exs b/test/archethic/self_repair/repair_worker_test.exs index 487520ae2..2d5bd2f3b 100644 --- a/test/archethic/self_repair/repair_worker_test.exs +++ b/test/archethic/self_repair/repair_worker_test.exs @@ -57,11 +57,11 @@ defmodule Archethic.SelfRepair.RepairWorkerTest do MockDB |> stub(:transaction_exists?, fn - "Bob2" -> + "Bob2", _ -> send(me, :exists_bob3) true - _ -> + _, _ -> false end) @@ -88,7 +88,7 @@ defmodule Archethic.SelfRepair.RepairWorkerTest do test "add_message/1 should add new addresses in GenServer state" do MockDB - |> stub(:transaction_exists?, fn _ -> Process.sleep(100) end) + |> stub(:transaction_exists?, fn _, _ -> Process.sleep(100) end) {:ok, pid} = RepairWorker.start_link( diff --git a/test/archethic/self_repair/sync/transaction_handler_test.exs b/test/archethic/self_repair/sync/transaction_handler_test.exs index 23eb71b6f..66848026a 100644 --- a/test/archethic/self_repair/sync/transaction_handler_test.exs +++ b/test/archethic/self_repair/sync/transaction_handler_test.exs @@ -173,7 +173,7 @@ defmodule Archethic.SelfRepair.Sync.TransactionHandlerTest do end) MockDB - |> stub(:write_transaction, fn ^tx -> + |> stub(:write_transaction, fn ^tx, _ -> send(me, :transaction_replicated) :ok end) diff --git a/test/archethic/self_repair/sync_test.exs b/test/archethic/self_repair/sync_test.exs index fe2b0b43f..cc0a40ffb 100644 --- a/test/archethic/self_repair/sync_test.exs +++ b/test/archethic/self_repair/sync_test.exs @@ -168,7 +168,7 @@ defmodule Archethic.SelfRepair.SyncTest do me = self() MockDB - |> stub(:write_transaction, fn ^tx -> + |> stub(:write_transaction, fn ^tx, _ -> send(me, :storage) :ok end) @@ -301,7 +301,7 @@ defmodule Archethic.SelfRepair.SyncTest do me = self() MockDB - |> stub(:write_transaction, fn ^transfer_tx -> + |> stub(:write_transaction, fn ^transfer_tx, _ -> send(me, :transaction_stored) :ok end) diff --git a/test/archethic/utils/detect_node_responsiveness_test.exs b/test/archethic/utils/detect_node_responsiveness_test.exs index 6de7122d8..a843c6dac 100644 --- a/test/archethic/utils/detect_node_responsiveness_test.exs +++ b/test/archethic/utils/detect_node_responsiveness_test.exs @@ -110,7 +110,7 @@ defmodule Archethic.Utils.DetectNodeResponsivenessTest do Process.monitor(pid) MockDB - |> stub(:transaction_exists?, fn ^address -> + |> stub(:transaction_exists?, fn ^address, _ -> false end) @@ -170,7 +170,7 @@ defmodule Archethic.Utils.DetectNodeResponsivenessTest do {:ok, pid} = DetectNodeResponsiveness.start_link(address, 2, replaying_fn, @timeout) MockDB - |> stub(:transaction_exists?, fn ^address -> + |> stub(:transaction_exists?, fn ^address, _ -> Process.send_after(me, :transaction_stored, 50) true end) @@ -221,10 +221,10 @@ defmodule Archethic.Utils.DetectNodeResponsivenessTest do {:ok, pid} = DetectNodeResponsiveness.start_link(address, 2, replaying_fn, @timeout) MockDB - |> expect(:transaction_exists?, fn ^address -> + |> expect(:transaction_exists?, fn ^address, _ -> false end) - |> expect(:transaction_exists?, fn ^address -> + |> expect(:transaction_exists?, fn ^address, _ -> Process.send_after(me, :transaction_stored, 50) true end) @@ -276,7 +276,7 @@ defmodule Archethic.Utils.DetectNodeResponsivenessTest do {:ok, pid} = DetectNodeResponsiveness.start_link(address, 2, replaying_fn, @timeout) MockDB - |> stub(:transaction_exists?, fn ^address -> + |> stub(:transaction_exists?, fn ^address, _ -> false end) @@ -327,7 +327,7 @@ defmodule Archethic.Utils.DetectNodeResponsivenessTest do {:ok, pid} = DetectNodeResponsiveness.start_link(address, 2, replaying_fn, @timeout) MockDB - |> stub(:transaction_exists?, fn ^address -> + |> stub(:transaction_exists?, fn ^address, _ -> false end) diff --git a/test/support/template.ex b/test/support/template.ex index c5713eeb6..98e7fa6f1 100644 --- a/test/support/template.ex +++ b/test/support/template.ex @@ -40,7 +40,7 @@ defmodule ArchethicCase do MockDB |> stub(:list_transactions, fn _ -> [] end) - |> stub(:write_transaction, fn _ -> :ok end) + |> stub(:write_transaction, fn _, _ -> :ok end) |> stub(:write_transaction_chain, fn _ -> :ok end) |> stub(:get_transaction, fn _, _ -> {:error, :transaction_not_exists} end) |> stub(:get_transaction_chain, fn _, _, _ -> {[], false, nil} end) @@ -56,7 +56,7 @@ defmodule ArchethicCase do |> stub(:count_transactions_by_type, fn _ -> 0 end) |> stub(:list_addresses_by_type, fn _ -> [] end) |> stub(:list_transactions, fn _ -> [] end) - |> stub(:transaction_exists?, fn _ -> false end) + |> stub(:transaction_exists?, fn _, _ -> false end) |> stub(:register_p2p_summary, fn _ -> :ok end) |> stub(:get_last_p2p_summaries, fn -> [] end) |> stub(:get_latest_tps, fn -> 0.0 end)