From 091d8e87676d9113f66b18a6febd48def2fbced7 Mon Sep 17 00:00:00 2001 From: Apoorv-2204 <90304197+apoorv-2204@users.noreply.github.com> Date: Fri, 2 Dec 2022 16:41:17 +0530 Subject: [PATCH 1/9] Fix self repair notifier (#645 ) * Fix notifier workflow for storage transaction * Add IO transactions * Add before? flag for self repair to avoid using the newly authorized nodes * Start Notifier after availability update to ensure nodes finished their self repair * Ensure addresses are expected to be replicated * Request missing chain addresses while new transaction stored * Fix beacon live downloading aggregate from BD instead of quorum * Store beacon aggregate when notifier is triggered Co-authored-by: Neylix --- Makefile | 5 +- config/config.exs | 1 + lib/archethic/beacon_chain.ex | 9 +- lib/archethic/beacon_chain/subset.ex | 7 +- lib/archethic/db.ex | 4 +- lib/archethic/db/embedded_impl.ex | 26 +- lib/archethic/db/embedded_impl/chain_index.ex | 51 +-- lib/archethic/p2p.ex | 42 +- lib/archethic/p2p/message.ex | 279 +++++++++----- lib/archethic/p2p/message/address_list.ex | 94 +++++ .../p2p/message/get_next_addresses.ex | 46 +++ lib/archethic/p2p/message/shard_repair.ex | 123 ++++++ .../p2p/message/transaction_input_list.ex | 4 +- lib/archethic/self_repair.ex | 83 +++- lib/archethic/self_repair/notifier.ex | 359 ++++++++++++------ lib/archethic/self_repair/repair_worker.ex | 184 +++++++++ lib/archethic/self_repair/supervisor.ex | 8 +- lib/archethic/self_repair/sync.ex | 37 +- lib/archethic/transaction_chain.ex | 63 ++- .../benchmarks/end_to_end_validation.ex | 1 - lib/archethic_web/live/chains/beacon_live.ex | 4 +- .../live/chains/node_shared_secrets_live.ex | 2 +- lib/archethic_web/live/chains/reward_live.ex | 2 +- .../p2p/message/address_list_test.exs | 7 + .../p2p/message/get_next_addresses_test.exs | 7 + .../p2p/message/shard_repair_test.exs | 7 + test/archethic/p2p/messages_test.exs | 137 ++++--- test/archethic/replication_test.exs | 2 +- test/archethic/self_repair/notifier_test.exs | 275 +++++++++++--- .../self_repair/repair_worker_test.exs | 164 ++++++++ test/archethic_web/live/rewards_live_test.exs | 1 - 31 files changed, 1612 insertions(+), 422 deletions(-) create mode 100644 lib/archethic/p2p/message/address_list.ex create mode 100644 lib/archethic/p2p/message/get_next_addresses.ex create mode 100644 lib/archethic/p2p/message/shard_repair.ex create mode 100644 lib/archethic/self_repair/repair_worker.ex create mode 100644 test/archethic/p2p/message/address_list_test.exs create mode 100644 test/archethic/p2p/message/get_next_addresses_test.exs create mode 100644 test/archethic/p2p/message/shard_repair_test.exs create mode 100644 test/archethic/self_repair/repair_worker_test.exs diff --git a/Makefile b/Makefile index b153deda3e..a839a63600 100644 --- a/Makefile +++ b/Makefile @@ -9,11 +9,11 @@ compile_c_programs: mkdir -p priv/c_dist $(CC) src/c/crypto/stdio_helpers.c src/c/crypto/ed25519.c -o priv/c_dist/libsodium_port -I src/c/crypto/stdio_helpers.h -lsodium $(CC) src/c/hypergeometric_distribution.c -o priv/c_dist/hypergeometric_distribution -lgmp - + git submodule update --force --recursive --init --remote $(MAKE) -C src/c/nat/miniupnp/miniupnpc cp src/c/nat/miniupnp/miniupnpc/build/upnpc-static priv/c_dist/upnpc - + ifeq ($(TPM_INSTALLED),0) $(CC) src/c/crypto/stdio_helpers.c src/c/crypto/tpm/lib.c src/c/crypto/tpm/port.c -o priv/c_dist/tpm_port -I src/c/crypto/stdio_helpers.h -I src/c/crypto/tpm/lib.h $(TPMFLAGS) @@ -24,6 +24,7 @@ endif clean: rm -f priv/c_dist/* mix archethic.clean_db + mix clean docker-clean: clean docker container stop $$(docker ps -a --filter=name=utn* -q) diff --git a/config/config.exs b/config/config.exs index 5c15f020d2..ba2457006b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -7,6 +7,7 @@ config :git_hooks, pre_push: [ tasks: [ {:cmd, "mix clean"}, + {:cmd, "mix git_hooks.install"}, {:cmd, "mix hex.outdated --within-requirements"}, {:cmd, "mix format --check-formatted"}, {:cmd, "mix compile --warnings-as-errors"}, diff --git a/lib/archethic/beacon_chain.ex b/lib/archethic/beacon_chain.ex index 7bf3f89c11..1215630d4e 100644 --- a/lib/archethic/beacon_chain.ex +++ b/lib/archethic/beacon_chain.ex @@ -320,17 +320,10 @@ defmodule Archethic.BeaconChain do end defp get_summary_address_by_node(date, subset, authorized_nodes) do - # Remove the newly authorized nodes at this specific time - filter_nodes = - case authorized_nodes do - [node] -> [node] - nodes -> Enum.filter(nodes, &(DateTime.compare(&1.authorization_date, date) == :lt)) - end - summary_address = Crypto.derive_beacon_chain_address(subset, date, true) subset - |> Election.beacon_storage_nodes(date, filter_nodes) + |> Election.beacon_storage_nodes(date, authorized_nodes) |> Enum.map(fn node -> {node, summary_address} end) diff --git a/lib/archethic/beacon_chain/subset.ex b/lib/archethic/beacon_chain/subset.ex index 1d374a58ea..129edc55c1 100644 --- a/lib/archethic/beacon_chain/subset.ex +++ b/lib/archethic/beacon_chain/subset.ex @@ -323,12 +323,7 @@ defmodule Archethic.BeaconChain.Subset do end defp broadcast_beacon_slot(subset, next_time, slot) do - # Remove the newly authorized nodes at this specific time - nodes = - case P2P.authorized_and_available_nodes(next_time) do - [node] -> [node] - nodes -> Enum.filter(nodes, &(DateTime.compare(&1.authorization_date, next_time) == :lt)) - end + nodes = P2P.authorized_and_available_nodes(next_time, true) subset |> Election.beacon_storage_nodes(next_time, nodes) diff --git a/lib/archethic/db.ex b/lib/archethic/db.ex index cc53a83e37..b638bd9aac 100644 --- a/lib/archethic/db.ex +++ b/lib/archethic/db.ex @@ -42,7 +42,7 @@ defmodule Archethic.DB do @callback list_addresses_by_type(Transaction.transaction_type()) :: Enumerable.t() | list(binary()) @callback list_chain_addresses(binary()) :: - Enumerable.t() | list({binary(), non_neg_integer()}) + Enumerable.t() | list({binary(), DateTime.t()}) @callback get_last_chain_address(binary()) :: {binary(), DateTime.t()} @callback get_last_chain_address(binary(), DateTime.t()) :: {binary(), DateTime.t()} @@ -79,4 +79,6 @@ defmodule Archethic.DB do :ok @callback get_inputs(ledger :: :UCO | :token, address :: binary()) :: list(VersionedTransactionInput.t()) + + @callback stream_first_addresses() :: Enumerable.t() end diff --git a/lib/archethic/db/embedded_impl.ex b/lib/archethic/db/embedded_impl.ex index 2ac7d99002..0136e78383 100644 --- a/lib/archethic/db/embedded_impl.ex +++ b/lib/archethic/db/embedded_impl.ex @@ -216,7 +216,7 @@ defmodule Archethic.DB.EmbeddedImpl do Stream all the addresses from the Genesis address(following it). """ @spec list_chain_addresses(binary()) :: - Enumerable.t() | list({binary(), non_neg_integer()}) + Enumerable.t() | list({binary(), DateTime.t()}) def list_chain_addresses(address) when is_binary(address) do ChainIndex.list_chain_addresses(address, db_path()) end @@ -249,16 +249,16 @@ defmodule Archethic.DB.EmbeddedImpl do end @doc """ - Reference a last address from a previous address + Reference a last address from a genesis address """ @spec add_last_transaction_address( - previous_address :: binary(), + genesis_address :: binary(), address :: binary(), tx_time :: DateTime.t() ) :: :ok - def add_last_transaction_address(previous_address, address, date = %DateTime{}) - when is_binary(previous_address) and is_binary(address) do - ChainIndex.set_last_chain_address(previous_address, address, date, db_path()) + def add_last_transaction_address(genesis_address, address, date = %DateTime{}) + when is_binary(genesis_address) and is_binary(address) do + ChainIndex.set_last_chain_address(genesis_address, address, date, db_path()) end @doc """ @@ -384,4 +384,18 @@ defmodule Archethic.DB.EmbeddedImpl do @spec get_inputs(ledger :: :UCO | :token, address :: binary()) :: list(VersionedTransactionInput.t()) defdelegate get_inputs(ledger, address), to: InputsReader, as: :get_inputs + + @doc """ + Stream first transactions address of a chain from genesis_address. + """ + @spec stream_first_addresses :: Enumerable.t() + def stream_first_addresses do + ChainIndex.list_genesis_addresses() + |> Stream.map(fn gen_address -> + gen_address + |> list_chain_addresses() + |> Enum.at(0) + |> elem(0) + end) + end end diff --git a/lib/archethic/db/embedded_impl/chain_index.ex b/lib/archethic/db/embedded_impl/chain_index.ex index 1dad19041e..1da38d4d8d 100644 --- a/lib/archethic/db/embedded_impl/chain_index.ex +++ b/lib/archethic/db/embedded_impl/chain_index.ex @@ -10,6 +10,11 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do alias Archethic.DB.EmbeddedImpl.ChainWriter alias Archethic.TransactionChain.Transaction + @archethic_db_tx_index :archethic_db_tx_index + @archethic_db_chain_stats :archethic_db_chain_stats + @archethic_db_last_index :archethic_db_last_index + @archethic_db_type_stats :archethic_db_type_stats + def start_link(arg \\ []) do GenServer.start_link(__MODULE__, arg, name: __MODULE__) end @@ -17,10 +22,10 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do def init(opts) do db_path = Keyword.fetch!(opts, :path) - :ets.new(:archethic_db_tx_index, [:set, :named_table, :public, read_concurrency: true]) - :ets.new(:archethic_db_chain_stats, [:set, :named_table, :public, read_concurrency: true]) - :ets.new(:archethic_db_last_index, [:set, :named_table, :public, read_concurrency: true]) - :ets.new(:archethic_db_type_stats, [:set, :named_table, :public, read_concurrency: true]) + :ets.new(@archethic_db_tx_index, [:set, :named_table, :public, read_concurrency: true]) + :ets.new(@archethic_db_chain_stats, [:set, :named_table, :public, read_concurrency: true]) + :ets.new(@archethic_db_last_index, [:set, :named_table, :public, read_concurrency: true]) + :ets.new(@archethic_db_type_stats, [:set, :named_table, :public, read_concurrency: true]) fill_tables(db_path) @@ -60,12 +65,12 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do true = :ets.insert( - :archethic_db_tx_index, + @archethic_db_tx_index, {current_address, %{size: size, offset: offset, genesis_address: genesis_address}} ) :ets.update_counter( - :archethic_db_chain_stats, + @archethic_db_chain_stats, genesis_address, [ {2, size}, @@ -89,7 +94,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do true = :ets.insert( - :archethic_db_last_index, + @archethic_db_last_index, {genesis_address, last_address, DateTime.to_unix(timestamp, :millisecond)} ) end) @@ -100,10 +105,10 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do case File.open(type_path(db_path, type), [:read, :binary]) do {:ok, fd} -> nb_txs = do_scan_types(fd) - :ets.insert(:archethic_db_type_stats, {type, nb_txs}) + :ets.insert(@archethic_db_type_stats, {type, nb_txs}) {:error, _} -> - :ets.insert(:archethic_db_type_stats, {type, 0}) + :ets.insert(@archethic_db_type_stats, {type, 0}) end end) end @@ -142,12 +147,12 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do # Write fast lookup entry for this transaction on memory true = :ets.insert( - :archethic_db_tx_index, + @archethic_db_tx_index, {tx_address, %{size: size, offset: last_offset, genesis_address: genesis_address}} ) :ets.update_counter( - :archethic_db_chain_stats, + @archethic_db_chain_stats, genesis_address, [ {2, size}, @@ -162,7 +167,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do @spec get_file_stats(binary()) :: {offset :: non_neg_integer(), nb_transactions :: non_neg_integer()} def get_file_stats(genesis_address) do - case :ets.lookup(:archethic_db_chain_stats, genesis_address) do + case :ets.lookup(@archethic_db_chain_stats, genesis_address) do [{_, last_offset, nb_txs}] -> {last_offset, nb_txs} @@ -207,7 +212,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do """ @spec get_tx_entry(binary(), String.t()) :: {:ok, map()} | {:error, :not_exists} def get_tx_entry(address, db_path) do - case :ets.lookup(:archethic_db_tx_index, address) do + case :ets.lookup(@archethic_db_tx_index, address) do [] -> # If the transaction is not found in the in memory lookup # we scan the index file for the subset of the transaction to find the relative information @@ -299,7 +304,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do Stream all the transaction addresses from genesis_address-address. """ @spec list_chain_addresses(binary(), String.t()) :: - Enumerable.t() | list({binary(), non_neg_integer()}) + Enumerable.t() | list({binary(), DateTime.t()}) def list_chain_addresses(address, db_path) when is_binary(address) do filepath = chain_addresses_path(db_path, address) @@ -316,7 +321,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do {:ok, hash} <- :file.read(fd, hash_size) do address = <> # return tuple of address and timestamp - {[{address, timestamp}], {:ok, fd}} + {[{address, DateTime.from_unix!(timestamp, :millisecond)}], {:ok, fd}} else :eof -> :file.close(fd) @@ -335,7 +340,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do """ @spec count_transactions_by_type(Transaction.transaction_type()) :: non_neg_integer() def count_transactions_by_type(type) do - case :ets.lookup(:archethic_db_type_stats, type) do + case :ets.lookup(@archethic_db_type_stats, type) do [] -> 0 @@ -351,7 +356,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do :ok def add_tx_type(type, address, db_path) do File.write!(type_path(db_path, type), address, [:append, :binary]) - :ets.update_counter(:archethic_db_type_stats, type, {2, 1}, {type, 0}) + :ets.update_counter(@archethic_db_type_stats, type, {2, 1}, {type, 0}) :ok end @@ -372,7 +377,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do filename = chain_addresses_path(db_path, genesis_address) write_last_chain_transaction? = - case :ets.lookup(:archethic_db_last_index, genesis_address) do + case :ets.lookup(@archethic_db_last_index, genesis_address) do [{_, ^new_address, _}] -> false [{_, _, chain_unix_time}] when unix_time < chain_unix_time -> false _ -> true @@ -380,7 +385,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do if write_last_chain_transaction? do :ok = File.write!(filename, encoded_data, [:binary, :append]) - true = :ets.insert(:archethic_db_last_index, {genesis_address, new_address, unix_time}) + true = :ets.insert(@archethic_db_last_index, {genesis_address, new_address, unix_time}) end :ok @@ -396,7 +401,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do case get_tx_entry(address, db_path) do {:ok, %{genesis_address: genesis_address}} -> # Search in the latest in memory index - case :ets.lookup(:archethic_db_last_index, genesis_address) do + case :ets.lookup(@archethic_db_last_index, genesis_address) do [] -> # If not present, the we search in the index file unix_time = DateTime.utc_now() |> DateTime.to_unix(:millisecond) @@ -410,7 +415,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do {:error, :not_exists} -> # We try if the request address is the genesis address to fetch the in memory index - case :ets.lookup(:archethic_db_last_index, address) do + case :ets.lookup(@archethic_db_last_index, address) do [] -> # If not present, the we search in the index file unix_time = DateTime.utc_now() |> DateTime.to_unix(:millisecond) @@ -610,14 +615,14 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do end defp stream_genesis_addresses(acc = []) do - case :ets.first(:archethic_db_chain_stats) do + case :ets.first(@archethic_db_chain_stats) do :"$end_of_table" -> {:halt, acc} first_key -> {[first_key], first_key} end end defp stream_genesis_addresses(acc) do - case :ets.next(:archethic_db_chain_stats, acc) do + case :ets.next(@archethic_db_chain_stats, acc) do :"$end_of_table" -> {:halt, acc} next_key -> {[next_key], next_key} end diff --git a/lib/archethic/p2p.ex b/lib/archethic/p2p.ex index 7c01e7ff2f..e6fe4c14cc 100644 --- a/lib/archethic/p2p.ex +++ b/lib/archethic/p2p.ex @@ -165,20 +165,25 @@ defmodule Archethic.P2P do end @doc """ - List the authorized nodes for the given datetime (default to now) + List the authorized nodes for the given datetime (default to now) or before if needed """ @spec authorized_nodes(DateTime.t()) :: list(Node.t()) - def authorized_nodes(date \\ DateTime.utc_now()) do + def authorized_nodes(date \\ DateTime.utc_now(), before? \\ false) do nodes = MemTable.authorized_nodes() |> Enum.filter(fn %Node{authorization_date: authorization_date} -> - DateTime.compare(authorization_date, date) != :gt + if before?, + do: DateTime.compare(authorization_date, date) == :lt, + else: DateTime.compare(authorization_date, date) != :gt end) case nodes do [] -> # Only happen during bootstrap - get_first_enrolled_node() + case get_first_enrolled_node() do + nil -> [] + node -> [node] + end nodes -> nodes @@ -187,34 +192,47 @@ defmodule Archethic.P2P do @doc """ List the authorized and available nodes + before? is used in for self repair to not take in account the newly + authorized nodes """ - @spec authorized_and_available_nodes(DateTime.t()) :: list(Node.t()) - def authorized_and_available_nodes(date \\ DateTime.utc_now()) do + @spec authorized_and_available_nodes(DateTime.t(), boolean()) :: list(Node.t()) + def authorized_and_available_nodes(date \\ DateTime.utc_now(), before? \\ false) do nodes = - authorized_nodes(date) + authorized_nodes(date, before?) |> Enum.filter(fn %Node{available?: true, availability_update: availability_update} -> - DateTime.compare(date, availability_update) != :lt + if before?, + do: DateTime.compare(date, availability_update) == :gt, + else: DateTime.compare(date, availability_update) != :lt %Node{available?: false, availability_update: availability_update} -> - DateTime.compare(date, availability_update) == :lt + if before?, + do: DateTime.compare(date, availability_update) != :gt, + else: DateTime.compare(date, availability_update) == :lt end) case nodes do [] -> # Only happen for init transactions so we take the first enrolled node - get_first_enrolled_node() + case get_first_enrolled_node() do + nil -> [] + node -> [node] + end nodes -> nodes end end - defp get_first_enrolled_node() do + @doc """ + Return the first enrolled node + """ + @spec get_first_enrolled_node() :: Node.t() | nil + def get_first_enrolled_node() do list_nodes() |> Enum.reject(&(&1.enrollment_date == nil)) |> Enum.sort_by(& &1.enrollment_date, {:asc, DateTime}) - |> Enum.take(1) + |> List.first() end @doc """ diff --git a/lib/archethic/p2p/message.ex b/lib/archethic/p2p/message.ex index b66dbe0fa0..140a309bfb 100644 --- a/lib/archethic/p2p/message.ex +++ b/lib/archethic/p2p/message.ex @@ -2,108 +2,109 @@ defmodule Archethic.P2P.Message do @moduledoc """ Provide functions to encode and decode P2P messages using a custom binary protocol """ - alias Archethic.Account - - alias Archethic.BeaconChain - alias Archethic.BeaconChain.ReplicationAttestation - alias Archethic.BeaconChain.Summary - alias Archethic.BeaconChain.SummaryAggregate - alias Archethic.BeaconChain.Slot - alias Archethic.BeaconChain.Subset - alias Archethic.BeaconChain.Slot - - alias Archethic.Contracts - - alias Archethic.Crypto - - alias Archethic.Election - - alias Archethic.Mining - - alias Archethic.P2P - - alias __MODULE__.AcknowledgeStorage - alias __MODULE__.AddMiningContext - alias __MODULE__.Balance - alias __MODULE__.BeaconSummaryList - alias __MODULE__.BeaconUpdate - alias __MODULE__.BootstrappingNodes - alias __MODULE__.CrossValidate - alias __MODULE__.CrossValidationDone - alias __MODULE__.EncryptedStorageNonce - alias __MODULE__.Error - alias __MODULE__.FirstPublicKey - alias __MODULE__.FirstAddress - alias __MODULE__.GetFirstAddress - alias __MODULE__.GetBalance - alias __MODULE__.GetBeaconSummaries - alias __MODULE__.GetBeaconSummary - alias __MODULE__.GetBeaconSummariesAggregate - alias __MODULE__.GetBootstrappingNodes - alias __MODULE__.GetCurrentSummaries - alias __MODULE__.GetFirstPublicKey - alias __MODULE__.GetLastTransaction - alias __MODULE__.GetLastTransactionAddress - alias __MODULE__.GetP2PView - alias __MODULE__.GetStorageNonce - alias __MODULE__.GetTransaction - alias __MODULE__.GetTransactionChain - alias __MODULE__.GetTransactionChainLength - alias __MODULE__.GetTransactionInputs - alias __MODULE__.GetTransactionSummary - alias __MODULE__.GetUnspentOutputs - alias __MODULE__.LastTransactionAddress - alias __MODULE__.ListNodes - alias __MODULE__.NewBeaconSlot - alias __MODULE__.NewTransaction - alias __MODULE__.NodeList - alias __MODULE__.NotFound - alias __MODULE__.NotifyEndOfNodeSync - alias __MODULE__.NotifyPreviousChain - alias __MODULE__.NotifyLastTransactionAddress - alias __MODULE__.Ok - alias __MODULE__.P2PView - alias __MODULE__.Ping - alias __MODULE__.RegisterBeaconUpdates - alias __MODULE__.ReplicateTransaction - alias __MODULE__.ReplicateTransactionChain - alias __MODULE__.ReplicationError - alias __MODULE__.StartMining - alias __MODULE__.TransactionChainLength - alias __MODULE__.TransactionInputList - alias __MODULE__.TransactionSummaryList - alias __MODULE__.TransactionList - alias __MODULE__.UnspentOutputList - alias __MODULE__.ValidationError - - alias Archethic.P2P.Node - - alias Archethic.PubSub - - alias Archethic.Replication - - alias Archethic.TransactionChain - alias Archethic.TransactionChain.Transaction - alias Archethic.TransactionChain.Transaction.CrossValidationStamp - alias Archethic.TransactionChain.Transaction.ValidationStamp - - alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations - - alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations.VersionedUnspentOutput - - alias Archethic.TransactionChain.TransactionInput - alias Archethic.TransactionChain.VersionedTransactionInput - alias Archethic.TransactionChain.TransactionSummary - - alias Archethic.TaskSupervisor - - alias Archethic.Utils - alias Archethic.Utils.VarInt - require Logger + alias Archethic.{ + Account, + BeaconChain, + Contracts, + Crypto, + Election, + Mining, + P2P, + P2P.Node, + PubSub, + Replication, + TransactionChain, + TaskSupervisor, + Utils, + Utils.VarInt + } + + alias Archethic.BeaconChain.{ + ReplicationAttestation, + Summary, + SummaryAggregate, + Slot, + Subset, + Slot + } + + alias Archethic.SelfRepair + + alias Archethic.TransactionChain.{ + Transaction, + Transaction.CrossValidationStamp, + Transaction.ValidationStamp, + TransactionInput, + TransactionSummary, + VersionedTransactionInput, + Transaction.ValidationStamp.LedgerOperations, + Transaction.ValidationStamp.LedgerOperations.VersionedUnspentOutput + } + + alias __MODULE__.{ + AcknowledgeStorage, + AddMiningContext, + AddressList, + Balance, + BeaconSummaryList, + BeaconUpdate, + BootstrappingNodes, + CrossValidate, + CrossValidationDone, + EncryptedStorageNonce, + Error, + FirstPublicKey, + FirstAddress, + GetFirstAddress, + GetBalance, + GetBeaconSummaries, + GetBeaconSummary, + GetBeaconSummariesAggregate, + GetBootstrappingNodes, + GetCurrentSummaries, + GetFirstPublicKey, + GetLastTransaction, + GetLastTransactionAddress, + GetNextAddresses, + GetP2PView, + GetStorageNonce, + GetTransaction, + GetTransactionChain, + GetTransactionChainLength, + GetTransactionInputs, + GetTransactionSummary, + GetUnspentOutputs, + LastTransactionAddress, + ListNodes, + NewBeaconSlot, + NewTransaction, + NodeList, + NotFound, + NotifyEndOfNodeSync, + NotifyLastTransactionAddress, + NotifyPreviousChain, + Ok, + P2PView, + Ping, + RegisterBeaconUpdates, + ReplicateTransaction, + ReplicateTransactionChain, + ReplicationError, + ShardRepair, + StartMining, + TransactionChainLength, + TransactionInputList, + TransactionSummaryList, + TransactionList, + UnspentOutputList, + ValidationError + } alias ArchethicWeb.TransactionSubscriber + require Logger + @type t :: request() | response() @type request :: @@ -141,6 +142,8 @@ defmodule Archethic.P2P.Message do | GetCurrentSummaries.t() | GetBeaconSummariesAggregate.t() | NotifyPreviousChain.t() + | ShardRepair.t() + | GetNextAddresses.t() @type response :: Ok.t() @@ -165,6 +168,7 @@ defmodule Archethic.P2P.Message do | FirstAddress.t() | ReplicationError.t() | SummaryAggregate.t() + | AddressList.t() @floor_upload_speed Application.compile_env!(:archethic, [__MODULE__, :floor_upload_speed]) @content_max_size Application.compile_env!(:archethic, :transaction_data_content_max_size) @@ -434,6 +438,18 @@ defmodule Archethic.P2P.Message do <<34::8, address::binary>> end + def encode(msg = %GetNextAddresses{}) do + <<35::8, GetNextAddresses.serialize(msg)::bitstring>> + end + + def encode(msg = %AddressList{}) do + <<229::8, AddressList.serialize(msg)::bitstring>> + end + + def encode(msg = %ShardRepair{}) do + <<230::8, ShardRepair.serialize(msg)::bitstring>> + end + def encode(aggregate = %SummaryAggregate{}) do <<231::8, SummaryAggregate.serialize(aggregate)::bitstring>> end @@ -968,6 +984,18 @@ defmodule Archethic.P2P.Message do {%NotifyPreviousChain{address: address}, rest} end + def decode(<<35::8, rest::bitstring>>) do + GetNextAddresses.deserialize(rest) + end + + def decode(<<229::8, rest::bitstring>>) do + AddressList.deserialize(rest) + end + + def decode(<<230::8, rest::bitstring>>) do + ShardRepair.deserialize(rest) + end + def decode(<<231::8, rest::bitstring>>) do SummaryAggregate.deserialize(rest) end @@ -1748,7 +1776,7 @@ defmodule Archethic.P2P.Message do def process(%NewBeaconSlot{slot: slot = %Slot{subset: subset, slot_time: slot_time}}, _) do summary_time = BeaconChain.next_summary_date(slot_time) - node_list = P2P.authorized_and_available_nodes(summary_time) + node_list = P2P.authorized_and_available_nodes(summary_time, true) beacon_summary_nodes = Election.beacon_storage_nodes( @@ -1835,6 +1863,59 @@ defmodule Archethic.P2P.Message do %Ok{} end + def process( + %ShardRepair{ + first_address: first_address, + storage_address: storage_address, + io_addresses: io_addresses + }, + _ + ) do + # Ensure all addresses are expected to be replicated + nodes = P2P.authorized_and_available_nodes() + + addresses = + if storage_address != nil, do: [storage_address | io_addresses], else: io_addresses + + public_key = Crypto.first_node_public_key() + + if Enum.all?( + addresses, + &(Election.storage_nodes(&1, nodes) |> Utils.key_in_node_list?(public_key)) + ) do + case SelfRepair.repair_in_progress?(first_address) do + false -> + SelfRepair.start_worker( + first_address: first_address, + storage_address: storage_address, + io_addresses: io_addresses + ) + + pid -> + SelfRepair.add_repair_addresses(pid, storage_address, io_addresses) + end + end + + %Ok{} + end + + def process(%GetNextAddresses{address: address}, _) do + case TransactionChain.get_transaction(address, validation_stamp: [:timestamp]) do + {:ok, %Transaction{validation_stamp: %ValidationStamp{timestamp: address_timestamp}}} -> + addresses = + TransactionChain.get_genesis_address(address) + |> TransactionChain.list_chain_addresses() + |> Enum.filter(fn {_address, timestamp} -> + DateTime.compare(timestamp, address_timestamp) == :gt + end) + + %AddressList{addresses: addresses} + + _ -> + %AddressList{addresses: []} + end + end + defp process_replication_chain(tx, replying_node_public_key) do Task.Supervisor.start_child(TaskSupervisor, fn -> response = diff --git a/lib/archethic/p2p/message/address_list.ex b/lib/archethic/p2p/message/address_list.ex new file mode 100644 index 0000000000..af0a09236a --- /dev/null +++ b/lib/archethic/p2p/message/address_list.ex @@ -0,0 +1,94 @@ +defmodule Archethic.P2P.Message.AddressList do + @moduledoc """ + Inform a shard to start repair. + """ + @enforce_keys [:addresses] + defstruct [:addresses] + + alias Archethic.Crypto + + alias Archethic.Utils + alias Archethic.Utils.VarInt + + @type t :: %__MODULE__{addresses: list(Crypto.prepended_hash())} + + @doc """ + Serialize AddressList Struct + + iex> %AddressList{ + ...> addresses: [{<<0, 0, 94, 5, 249, 103, 126, 31, 43, 57, 25, 14, 187, 133, 59, 234, 201, 172, + ...> 3, 195, 43, 81, 81, 146, 164, 202, 147, 218, 207, 204, 31, 185, 73, 251>>, ~U[2022-11-27 12:34:56.789Z]}, + ...> {<<0, 0, 94, 5, 249, 103, 126, 31, 43, 57, 25, 14, 187, 133, 59, 234, 201, 172, + ...> 3, 195, 43, 81, 81, 146, 164, 202, 147, 218, 207, 204, 31, 185, 123, 321>>, ~U[2022-11-27 12:34:54.321Z]}] + ...> } |> AddressList.serialize() + # VarInt + <<1, 2, + # Addresses + 0, 0, 94, 5, 249, 103, 126, 31, 43, 57, 25, 14, 187, 133, 59, 234, 201, 172, + 3, 195, 43, 81, 81, 146, 164, 202, 147, 218, 207, 204, 31, 185, 73, 251, + 0, 0, 1, 132, 185, 21, 96, 149, + 0, 0, 94, 5, 249, 103, 126, 31, 43, 57, 25, 14, 187, 133, 59, 234, 201, 172, + 3, 195, 43, 81, 81, 146, 164, 202, 147, 218, 207, 204, 31, 185, 123, 321, + 0, 0, 1, 132, 185, 21, 86, 241>> + """ + def serialize(%__MODULE__{addresses: addresses}) do + addresses_bin = + addresses + |> Stream.map(fn {address, timestamp} -> + <> + end) + |> Enum.to_list() + |> :erlang.list_to_binary() + + <> + end + + @doc """ + Deserialize AddressList Struct + + iex> # VarInt + ...> <<1, 2, + ...> # Addresses + ...> 0, 0, 94, 5, 249, 103, 126, 31, 43, 57, 25, 14, 187, 133, 59, 234, 201, 172, + ...> 3, 195, 43, 81, 81, 146, 164, 202, 147, 218, 207, 204, 31, 185, 73, 251, + ...> 0, 0, 1, 132, 185, 21, 96, 149, + ...> 0, 0, 94, 5, 249, 103, 126, 31, 43, 57, 25, 14, 187, 133, 59, 234, 201, 172, + ...> 3, 195, 43, 81, 81, 146, 164, 202, 147, 218, 207, 204, 31, 185, 123, 321, + ...> 0, 0, 1, 132, 185, 21, 86, 241>> + ...> |> AddressList.deserialize() + { + %AddressList{ + addresses: [ + {<<0, 0, 94, 5, 249, 103, 126, 31, 43, 57, 25, 14, 187, 133, 59, 234, 201, 172, + 3, 195, 43, 81, 81, 146, 164, 202, 147, 218, 207, 204, 31, 185, 73, 251>>, ~U[2022-11-27 12:34:56.789Z]}, + {<<0, 0, 94, 5, 249, 103, 126, 31, 43, 57, 25, 14, 187, 133, 59, 234, 201, 172, + 3, 195, 43, 81, 81, 146, 164, 202, 147, 218, 207, 204, 31, 185, 123, 321>>, ~U[2022-11-27 12:34:54.321Z]} + ] + }, ""} + + """ + def deserialize(bin) do + {addresses_length, rest} = VarInt.get_value(bin) + + {addresses, rest} = deserialize_list(rest, addresses_length, []) + + {%__MODULE__{addresses: addresses}, rest} + end + + defp deserialize_list(rest, 0, _), do: {[], rest} + + defp deserialize_list(rest, nb_elt, acc) when length(acc) == nb_elt do + {Enum.reverse(acc), rest} + end + + defp deserialize_list(rest, nb_elt, acc) do + {elt, rest} = deserialize_elt(rest) + deserialize_list(rest, nb_elt, [elt | acc]) + end + + defp deserialize_elt(bin) do + {address, <>} = Utils.deserialize_address(bin) + + {{address, DateTime.from_unix!(timestamp, :millisecond)}, rest} + end +end diff --git a/lib/archethic/p2p/message/get_next_addresses.ex b/lib/archethic/p2p/message/get_next_addresses.ex new file mode 100644 index 0000000000..87e3e3c22b --- /dev/null +++ b/lib/archethic/p2p/message/get_next_addresses.ex @@ -0,0 +1,46 @@ +defmodule Archethic.P2P.Message.GetNextAddresses do + @moduledoc """ + Inform a shard to start repair. + """ + @enforce_keys [:address] + defstruct [:address] + + alias Archethic.Crypto + + alias Archethic.Utils + + @type t :: %__MODULE__{address: Crypto.prepended_hash()} + + @doc """ + Serialize GetNextAddresses Struct + + iex> %GetNextAddresses{ + ...> address: <<0, 0, 94, 5, 249, 103, 126, 31, 43, 57, 25, 14, 187, 133, 59, 234, 201, 172, + ...> 3, 195, 43, 81, 81, 146, 164, 202, 147, 218, 207, 204, 31, 185, 73, 251>> + ...> } |> GetNextAddresses.serialize() + <<0, 0, 94, 5, 249, 103, 126, 31, 43, 57, 25, 14, 187, 133, 59, 234, 201, 172, + 3, 195, 43, 81, 81, 146, 164, 202, 147, 218, 207, 204, 31, 185, 73, 251>> + """ + def serialize(%__MODULE__{address: address}) do + <> + end + + @doc """ + Deserialize GetNextAddresses Struct + + iex> <<0, 0, 94, 5, 249, 103, 126, 31, 43, 57, 25, 14, 187, 133, 59, 234, 201, 172, + ...> 3, 195, 43, 81, 81, 146, 164, 202, 147, 218, 207, 204, 31, 185, 73, 251>> + ...> |> GetNextAddresses.deserialize() + { + %GetNextAddresses{ + address: <<0, 0, 94, 5, 249, 103, 126, 31, 43, 57, 25, 14, 187, 133, 59, 234, 201, 172, + 3, 195, 43, 81, 81, 146, 164, 202, 147, 218, 207, 204, 31, 185, 73, 251>> + }, ""} + + """ + def deserialize(bin) do + {address, rest} = Utils.deserialize_address(bin) + + {%__MODULE__{address: address}, rest} + end +end diff --git a/lib/archethic/p2p/message/shard_repair.ex b/lib/archethic/p2p/message/shard_repair.ex new file mode 100644 index 0000000000..e714267b8e --- /dev/null +++ b/lib/archethic/p2p/message/shard_repair.ex @@ -0,0 +1,123 @@ +defmodule Archethic.P2P.Message.ShardRepair do + @moduledoc """ + Inform a shard to start repair. + """ + @enforce_keys [:first_address, :storage_address, :io_addresses] + defstruct [:first_address, :storage_address, :io_addresses] + + alias Archethic.Crypto + + alias Archethic.Utils + alias Archethic.Utils.VarInt + + @type t :: %__MODULE__{ + first_address: Crypto.prepended_hash(), + storage_address: Crypto.prepended_hash(), + io_addresses: list(Crypto.prepended_hash()) + } + + @doc """ + Serialize ShardRepair Struct + + iex> %ShardRepair{ + ...> first_address: <<0, 0, 94, 5, 249, 103, 126, 31, 43, 57, 25, 14, 187, 133, 59, 234, 201, 172, + ...> 3, 195, 43, 81, 81, 146, 164, 202, 147, 218, 207, 204, 31, 185, 73, 251>>, + ...> storage_address: <<0, 0, 106, 248, 193, 217, 112, 140, 200, 141, 33, 81, 243, 92, 207, 242, 72, + ...> 2, 92, 236, 236, 100, 121, 250, 105, 12, 90, 240, 221, 108, 1, 171, 108, 130>>, + ...> io_addresses: [ + ...> <<0, 0, 106, 248, 193, 217, 112, 140, 200, 141, 33, 81, 243, 92, 207, 242, 72, + ...> 2, 92, 236, 236, 100, 121, 250, 105, 12, 90, 240, 221, 108, 1, 171, 108, 130>>, + ...> <<0, 0, 106, 248, 193, 217, 112, 140, 200, 141, 33, 81, 243, 92, 207, 242, 72, + ...> 2, 92, 236, 236, 100, 121, 250, 105, 12, 90, 240, 221, 108, 1, 171, 108, 130>> + ...> ] + ...> } |> ShardRepair.serialize() + # First address + <<0, 0, 94, 5, 249, 103, 126, 31, 43, 57, 25, 14, 187, 133, 59, 234, 201, 172, + 3, 195, 43, 81, 81, 146, 164, 202, 147, 218, 207, 204, 31, 185, 73, 251, + # Storage address? + 1::1, + # Storage address + 0, 0, 106, 248, 193, 217, 112, 140, 200, 141, 33, 81, 243, 92, 207, 242, 72, + 2, 92, 236, 236, 100, 121, 250, 105, 12, 90, 240, 221, 108, 1, 171, 108, 130, + # Varint + 1, 2, + #IO addresses + 0, 0, 106, 248, 193, 217, 112, 140, 200, 141, 33, 81, 243, 92, 207, 242, 72, + 2, 92, 236, 236, 100, 121, 250, 105, 12, 90, 240, 221, 108, 1, 171, 108, 130, + 0, 0, 106, 248, 193, 217, 112, 140, 200, 141, 33, 81, 243, 92, 207, 242, 72, + 2, 92, 236, 236, 100, 121, 250, 105, 12, 90, 240, 221, 108, 1, 171, 108, 130>> + """ + def serialize(%__MODULE__{ + first_address: first_address, + storage_address: nil, + io_addresses: io_addresses + }) do + <> + end + + def serialize(%__MODULE__{ + first_address: first_address, + storage_address: storage_address, + io_addresses: io_addresses + }) do + <> + end + + @doc """ + DeSerialize ShardRepair Struct + + iex> # First address + ...> <<0, 0, 94, 5, 249, 103, 126, 31, 43, 57, 25, 14, 187, 133, 59, 234, 201, 172, + ...> 3, 195, 43, 81, 81, 146, 164, 202, 147, 218, 207, 204, 31, 185, 73, 251, + ...> # Storage address? + ...> 1::1, + ...> # Storage address + ...> 0, 0, 106, 248, 193, 217, 112, 140, 200, 141, 33, 81, 243, 92, 207, 242, 72, + ...> 2, 92, 236, 236, 100, 121, 250, 105, 12, 90, 240, 221, 108, 1, 171, 108, 130, + ...> # Varint + ...> 1, 2, + ...> #IO addresses + ...> 0, 0, 106, 248, 193, 217, 112, 140, 200, 141, 33, 81, 243, 92, 207, 242, 72, + ...> 2, 92, 236, 236, 100, 121, 250, 105, 12, 90, 240, 221, 108, 1, 171, 108, 130, + ...> 0, 0, 106, 248, 193, 217, 112, 140, 200, 141, 33, 81, 243, 92, 207, 242, 72, + ...> 2, 92, 236, 236, 100, 121, 250, 105, 12, 90, 240, 221, 108, 1, 171, 108, 130>> + ...> |> ShardRepair.deserialize() + { + %ShardRepair{ + first_address: <<0, 0, 94, 5, 249, 103, 126, 31, 43, 57, 25, 14, 187, 133, 59, 234, 201, 172, + 3, 195, 43, 81, 81, 146, 164, 202, 147, 218, 207, 204, 31, 185, 73, 251>>, + storage_address: <<0, 0, 106, 248, 193, 217, 112, 140, 200, 141, 33, 81, 243, 92, 207, 242, 72, + 2, 92, 236, 236, 100, 121, 250, 105, 12, 90, 240, 221, 108, 1, 171, 108, 130>>, + io_addresses: [ + <<0, 0, 106, 248, 193, 217, 112, 140, 200, 141, 33, 81, 243, 92, 207, 242, 72, + 2, 92, 236, 236, 100, 121, 250, 105, 12, 90, 240, 221, 108, 1, 171, 108, 130>>, + <<0, 0, 106, 248, 193, 217, 112, 140, 200, 141, 33, 81, 243, 92, 207, 242, 72, + 2, 92, 236, 236, 100, 121, 250, 105, 12, 90, 240, 221, 108, 1, 171, 108, 130>> + ] + }, ""} + + """ + def deserialize(bin) do + {first_address, <>} = Utils.deserialize_address(bin) + + {storage_address, rest} = + if storage_address? == 1 do + Utils.deserialize_address(rest) + else + {nil, rest} + end + + {io_addresses_length, rest} = VarInt.get_value(rest) + + {io_addresses, rest} = Utils.deserialize_addresses(rest, io_addresses_length, []) + + {%__MODULE__{ + first_address: first_address, + storage_address: storage_address, + io_addresses: io_addresses + }, rest} + end +end diff --git a/lib/archethic/p2p/message/transaction_input_list.ex b/lib/archethic/p2p/message/transaction_input_list.ex index afac9d71c1..0da664eee5 100644 --- a/lib/archethic/p2p/message/transaction_input_list.ex +++ b/lib/archethic/p2p/message/transaction_input_list.ex @@ -4,10 +4,10 @@ defmodule Archethic.P2P.Message.TransactionInputList do """ defstruct inputs: [], more?: false, offset: 0 - alias Archethic.TransactionChain.TransactionInput + alias Archethic.TransactionChain.VersionedTransactionInput @type t() :: %__MODULE__{ - inputs: list(TransactionInput.t()), + inputs: list(VersionedTransactionInput.t()), more?: boolean(), offset: non_neg_integer() } diff --git a/lib/archethic/self_repair.ex b/lib/archethic/self_repair.ex index 45dd86f8db..020e8a2e3d 100755 --- a/lib/archethic/self_repair.ex +++ b/lib/archethic/self_repair.ex @@ -4,11 +4,19 @@ defmodule Archethic.SelfRepair do the bootstrapping phase and stores last synchronization date after each cycle. """ + alias __MODULE__.Notifier + alias __MODULE__.NotifierSupervisor + alias __MODULE__.RepairRegistry + alias __MODULE__.RepairWorker alias __MODULE__.Scheduler alias __MODULE__.Sync alias Archethic.BeaconChain + alias Archethic.Crypto + + alias Archethic.P2P.Node + alias Crontab.CronExpression.Parser, as: CronParser alias Crontab.Scheduler, as: CronScheduler @@ -67,12 +75,6 @@ defmodule Archethic.SelfRepair do @spec put_last_sync_date(DateTime.t()) :: :ok defdelegate put_last_sync_date(datetime), to: Sync, as: :store_last_sync_date - def config_change(changed_conf) do - changed_conf - |> Keyword.get(Scheduler) - |> Scheduler.config_change() - end - @doc """ Return the previous scheduler time from a given date """ @@ -83,4 +85,73 @@ defmodule Archethic.SelfRepair do |> CronScheduler.get_previous_run_date!(DateTime.to_naive(date_from)) |> DateTime.from_naive!("Etc/UTC") end + + @doc """ + Start a new notifier process if there is new unavailable nodes after the self repair + """ + @spec start_notifier(list(Node.t()), list(Node.t()), DateTime.t()) :: :ok + def start_notifier(prev_available_nodes, new_available_nodes, availability_update) do + diff_node = + (prev_available_nodes -- new_available_nodes) + |> Enum.reject(&(&1.first_public_key == Crypto.first_node_public_key())) + + case diff_node do + [] -> + :ok + + nodes -> + unavailable_nodes = Enum.map(nodes, & &1.first_public_key) + + DynamicSupervisor.start_child( + NotifierSupervisor, + {Notifier, + unavailable_nodes: unavailable_nodes, + prev_available_nodes: prev_available_nodes, + new_available_nodes: new_available_nodes, + availability_update: availability_update} + ) + + :ok + end + end + + @doc """ + Return pid of a running RepairWorker for the first_address, or false + """ + @spec repair_in_progress?(first_address :: binary()) :: false | pid() + def repair_in_progress?(first_address) do + case Registry.lookup(RepairRegistry, first_address) do + [{pid, _}] -> + pid + + _ -> + false + end + end + + @doc """ + Start a new RepairWorker for the first_address + """ + @spec start_worker(list()) :: DynamicSupervisor.on_start_child() + def start_worker(args) do + DynamicSupervisor.start_child(NotifierSupervisor, {RepairWorker, args}) + end + + @doc """ + Add a new address in the address list of the RepairWorker + """ + @spec add_repair_addresses( + pid(), + Crypto.prepended_hash() | nil, + list(Crypto.prepended_hash()) + ) :: :ok + def add_repair_addresses(pid, storage_address, io_addresses) do + GenServer.cast(pid, {:add_address, storage_address, io_addresses}) + end + + def config_change(changed_conf) do + changed_conf + |> Keyword.get(Scheduler) + |> Scheduler.config_change() + end end diff --git a/lib/archethic/self_repair/notifier.ex b/lib/archethic/self_repair/notifier.ex index 2463ef9105..886727aefc 100644 --- a/lib/archethic/self_repair/notifier.ex +++ b/lib/archethic/self_repair/notifier.ex @@ -16,158 +16,295 @@ defmodule Archethic.SelfRepair.Notifier do H -->|Replicate Transaction| C[Node2] H -->|Replicate Transaction| D[Node3] ``` - """ - use GenServer - @vsn Mix.Project.config()[:version] + alias Archethic.{ + BeaconChain, + Crypto, + Election, + P2P, + P2P.Node, + P2P.Message.ShardRepair, + TransactionChain, + TransactionChain.Transaction, + Utils + } - alias Archethic.Crypto + alias Archethic.TransactionChain.Transaction.{ + ValidationStamp, + ValidationStamp.LedgerOperations + } - alias Archethic.Election + use GenServer, restart: :temporary - alias Archethic.PubSub + require Logger - alias Archethic.P2P - alias Archethic.P2P.Message.ReplicateTransaction - alias Archethic.P2P.Node + def start_link(args) do + GenServer.start_link(__MODULE__, args) + end - alias Archethic.TaskSupervisor + def init(args) do + availability_update = Keyword.fetch!(args, :availability_update) - alias Archethic.TransactionChain - alias Archethic.TransactionChain.Transaction - alias Archethic.TransactionChain.Transaction.ValidationStamp + seconds = DateTime.diff(availability_update, DateTime.utc_now()) - alias Archethic.Utils + if seconds > 0 do + Process.send_after(self(), :start, seconds * 1000) + else + send(self(), :start) + end - require Logger + {:ok, args} + end - def start_link(args \\ []) do - GenServer.start_link(__MODULE__, args) + def handle_info(:start, data) do + unavailable_nodes = Keyword.fetch!(data, :unavailable_nodes) + prev_available_nodes = Keyword.fetch!(data, :prev_available_nodes) + new_available_nodes = Keyword.fetch!(data, :new_available_nodes) + + Logger.info( + "Start Notifier due to a topology change #{inspect(Enum.map(unavailable_nodes, &Base.encode16(&1)))}" + ) + + repair_transactions(unavailable_nodes, prev_available_nodes, new_available_nodes) + repair_summaries_aggregate(prev_available_nodes, new_available_nodes) + + {:stop, :normal, data} end - def init(_) do - PubSub.register_to_node_update() - {:ok, %{notified: %{}}} + @doc """ + For each txn chain in db. Load its genesis address, load its + chain, recompute shards , notifiy nodes. Network txns are excluded. + """ + @spec repair_transactions(list(Crypto.key()), list(Node.t()), list(Node.t())) :: :ok + def repair_transactions(unavailable_nodes, prev_available_nodes, new_available_nodes) do + # We fetch all the transactions existing and check if the disconnected nodes were in storage nodes + TransactionChain.stream_first_addresses() + |> Stream.reject(&network_chain?(&1)) + |> Stream.chunk_every(20) + |> Stream.each(fn chunk -> + concurrent_txn_processing( + chunk, + unavailable_nodes, + prev_available_nodes, + new_available_nodes + ) + end) + |> Stream.run() end - def handle_info( - {:node_update, - %Node{ - available?: false, - authorized?: true, - first_public_key: node_key, - authorization_date: authorization_date - }}, - state = %{notified: notified} - ) do - current_node_public_key = Crypto.first_node_public_key() - now = DateTime.utc_now() |> DateTime.truncate(:millisecond) - - with :lt <- DateTime.compare(authorization_date, now), - nil <- Map.get(notified, node_key), - false <- current_node_public_key == node_key do - repair_transactions(node_key, current_node_public_key) - {:noreply, Map.update!(state, :notified, &Map.put(&1, node_key, %{}))} - else + defp network_chain?(address) do + case TransactionChain.get_transaction(address, [:type]) do + {:ok, %Transaction{type: type}} -> + Transaction.network_type?(type) + _ -> - {:noreply, state} + false end end - def handle_info( - {:node_update, - %Node{authorized?: false, authorization_date: date, first_public_key: node_key}}, - state = %{notified: notified} - ) - when date != nil do - current_node_public_key = Crypto.first_node_public_key() + defp concurrent_txn_processing( + addresses, + unavailable_nodes, + prev_available_nodes, + new_available_nodes + ) do + Task.Supervisor.async_stream_nolink( + Archethic.TaskSupervisor, + addresses, + &sync_chain(&1, unavailable_nodes, prev_available_nodes, new_available_nodes), + ordered: false, + on_timeout: :kill_task + ) + |> Stream.run() + end - with nil <- Map.get(notified, node_key), - false <- current_node_public_key == node_key do - repair_transactions(node_key, current_node_public_key) - {:noreply, Map.update!(state, :notified, &Map.put(&1, node_key, %{}))} - else - _ -> - {:noreply, state} - end + defp sync_chain(address, unavailable_nodes, prev_available_nodes, new_available_nodes) do + address + |> TransactionChain.stream([ + :address, + validation_stamp: [ledger_operations: [:transaction_movements]] + ]) + |> Stream.map(&get_previous_election(&1, prev_available_nodes)) + |> Stream.filter(&storage_or_io_node?(&1, unavailable_nodes)) + |> Stream.filter(¬ify?(&1)) + |> Stream.map(&new_storage_nodes(&1, new_available_nodes)) + |> map_last_addresses_for_node() + |> notify_nodes(address) end - def handle_info( - {:node_update, - %Node{available?: true, first_public_key: node_key, authorization_date: date}}, - state - ) - when date != nil do - {:noreply, Map.update!(state, :notified, &Map.delete(&1, node_key))} + defp get_previous_election( + %Transaction{ + address: address, + validation_stamp: %ValidationStamp{ + ledger_operations: %LedgerOperations{transaction_movements: transaction_movements} + } + }, + prev_available_nodes + ) do + prev_storage_nodes = + Election.chain_storage_nodes(address, prev_available_nodes) + |> Enum.map(& &1.first_public_key) + + resolved_addresses = + transaction_movements + |> Enum.map(& &1.to) + + prev_io_nodes = + resolved_addresses + |> Election.io_storage_nodes(prev_available_nodes) + |> Enum.map(& &1.first_public_key) + + {address, resolved_addresses, prev_storage_nodes, prev_io_nodes -- prev_storage_nodes} end - def handle_info(_, state) do - {:noreply, state} + defp storage_or_io_node?({_, _, prev_storage_nodes, prev_io_nodes}, unavailable_nodes) do + nodes = prev_storage_nodes ++ prev_io_nodes + Enum.any?(unavailable_nodes, &Enum.member?(nodes, &1)) end - defp repair_transactions(node_key, current_node_public_key) do - Logger.debug("Trying to repair transactions due to a topology change", - node: Base.encode16(node_key) - ) + # Notify only if the current node is part of the previous storage / io nodes + # to reduce number of messages + defp notify?({_, _, prev_storage_nodes, prev_io_nodes}) do + Enum.member?(prev_storage_nodes ++ prev_io_nodes, Crypto.first_node_public_key()) + end - node_key - |> get_transactions_to_sync() - |> Stream.each(&forward_transaction(&1, current_node_public_key)) - |> Stream.run() + @doc """ + New election is carried out on the set of all authorized omiting unavailable_node. + The set of previous storage nodes is subtracted from the set of new storage nodes. + """ + @spec new_storage_nodes( + {binary(), list(Crypto.prepended_hash()), list(Crypto.key()), list(Crypto.key())}, + list(Node.t()) + ) :: + {binary(), list(Crypto.key()), list(Crypto.key())} + def new_storage_nodes( + {address, resolved_addresses, prev_storage_nodes, prev_io_nodes}, + new_available_nodes + ) do + new_storage_nodes = + Election.chain_storage_nodes(address, new_available_nodes) + |> Enum.map(& &1.first_public_key) + |> Enum.reject(&Enum.member?(prev_storage_nodes, &1)) + + already_stored_nodes = prev_storage_nodes ++ prev_io_nodes ++ new_storage_nodes + + new_io_nodes = + resolved_addresses + |> Election.io_storage_nodes(new_available_nodes) + |> Enum.map(& &1.first_public_key) + |> Enum.reject(&Enum.member?(already_stored_nodes, &1)) + + {address, new_storage_nodes, new_io_nodes} end - defp get_transactions_to_sync(node_public_key) do - # We fetch all the transactions existing and check if the disconnecting node was a storage node - TransactionChain.list_all([:address, :type, validation_stamp: [:timestamp]]) - |> Stream.map( - fn tx = %Transaction{ - address: address, - type: type, - validation_stamp: %ValidationStamp{timestamp: timestamp} - } -> - node_list = - Enum.filter( - P2P.list_nodes(), - &(&1.authorization_date != nil and - DateTime.compare(&1.authorization_date, timestamp) == :lt) - ) + @doc """ + Create a map returning for each node the last transaction address it should replicate + """ + @spec map_last_addresses_for_node(Enumerable.t()) :: Enumerable.t() + def map_last_addresses_for_node(stream) do + Enum.reduce( + stream, + %{}, + fn {address, new_storage_nodes, new_io_nodes}, acc -> + acc = + Enum.reduce(new_storage_nodes, acc, fn first_public_key, acc -> + Map.update( + acc, + first_public_key, + %{last_address: address, io_addresses: []}, + &Map.put(&1, :last_address, address) + ) + end) - {tx, Election.chain_storage_nodes_with_type(address, type, node_list)} + Enum.reduce(new_io_nodes, acc, fn first_public_key, acc -> + Map.update( + acc, + first_public_key, + %{last_address: nil, io_addresses: [address]}, + &Map.update(&1, :io_addresses, [address], fn addresses -> [address | addresses] end) + ) + end) end ) - |> Stream.filter(fn {_tx, nodes} -> - Utils.key_in_node_list?(nodes, node_public_key) - end) end - defp forward_transaction( - {tx = %Transaction{address: address, type: type}, previous_storage_nodes}, - current_node_public_key - ) do - # We compute the new storage nodes minus the previous ones - new_storage_nodes = - Election.chain_storage_nodes_with_type( - address, - type, - P2P.authorized_nodes() -- previous_storage_nodes - ) + defp notify_nodes(acc, first_address) do + Task.Supervisor.async_stream_nolink( + Archethic.TaskSupervisor, + acc, + fn {node_first_public_key, %{last_address: last_address, io_addresses: io_addresses}} -> + Logger.info( + "Send Shard Repair message to #{Base.encode16(node_first_public_key)}" <> + "with storage_address #{if last_address, do: Base.encode16(last_address), else: nil}, " <> + "io_addresses #{inspect(Enum.map(io_addresses, &Base.encode16(&1)))}", + address: Base.encode16(first_address) + ) - with false <- Enum.empty?(new_storage_nodes), - true <- Utils.key_in_node_list?(previous_storage_nodes, current_node_public_key) do - Logger.info("Repair started due to network topology change", - transaction_address: Base.encode16(address), - transaction_type: type - ) + P2P.send_message(node_first_public_key, %ShardRepair{ + first_address: first_address, + storage_address: last_address, + io_addresses: io_addresses + }) + end, + ordered: false, + on_timeout: :kill_task + ) + |> Stream.run() + end + @doc """ + For each beacon aggregate, calculate the new election and store it if the node needs to + """ + @spec repair_summaries_aggregate(list(Node.t()), list(Node.t())) :: :ok + def repair_summaries_aggregate(prev_available_nodes, new_available_nodes) do + %Node{enrollment_date: first_enrollment_date} = P2P.get_first_enrolled_node() + + first_enrollment_date + |> BeaconChain.next_summary_dates() + |> Stream.filter(&download?(&1, new_available_nodes)) + |> Stream.chunk_every(20) + |> Stream.each(fn summary_times -> Task.Supervisor.async_stream_nolink( - TaskSupervisor, - new_storage_nodes, - &P2P.send_message(&1, %ReplicateTransaction{transaction: tx}), + Archethic.TaskSupervisor, + summary_times, + &download_and_store_summary(&1, prev_available_nodes), ordered: false, on_timeout: :kill_task ) |> Stream.run() + end) + |> Stream.run() + end + + defp download?(summary_time, new_available_nodes) do + in_new_election? = + summary_time + |> Crypto.derive_beacon_aggregate_address() + |> Election.chain_storage_nodes(new_available_nodes) + |> Utils.key_in_node_list?(Crypto.first_node_public_key()) + + if in_new_election? do + case BeaconChain.get_summaries_aggregate(summary_time) do + {:ok, _} -> false + {:error, _} -> true + end + else + false + end + end + + defp download_and_store_summary(summary_time, prev_available_nodes) do + case BeaconChain.fetch_summaries_aggregate(summary_time, prev_available_nodes) do + {:ok, aggregate} -> + Logger.debug("Notifier store beacon aggregate for #{summary_time}") + BeaconChain.write_summaries_aggregate(aggregate) + + error -> + Logger.warning( + "Notifier cannot fetch summary aggregate for date #{summary_time} " <> + "because of #{inspect(error)}" + ) end end end diff --git a/lib/archethic/self_repair/repair_worker.ex b/lib/archethic/self_repair/repair_worker.ex new file mode 100644 index 0000000000..d2113bd331 --- /dev/null +++ b/lib/archethic/self_repair/repair_worker.ex @@ -0,0 +1,184 @@ +defmodule Archethic.SelfRepair.RepairWorker do + @moduledoc false + + alias Archethic.{ + Contracts, + BeaconChain, + Election, + P2P, + Replication, + TransactionChain + } + + alias Archethic.SelfRepair.RepairRegistry + + use GenServer, restart: :transient + + require Logger + + def start_link(args) do + GenServer.start_link(__MODULE__, args, []) + end + + def init(args) do + first_address = Keyword.fetch!(args, :first_address) + storage_address = Keyword.fetch!(args, :storage_address) + io_addresses = Keyword.fetch!(args, :io_addresses) + + Registry.register(RepairRegistry, first_address, []) + + Logger.info( + "Notifier Repair Worker start with storage_address #{if storage_address, do: Base.encode16(storage_address), else: nil}, " <> + "io_addresses #{inspect(Enum.map(io_addresses, &Base.encode16(&1)))}", + address: Base.encode16(first_address) + ) + + # We get the authorized nodes before the last summary date as we are sure that they know + # the informations we need. Requesting current nodes may ask information to nodes in same repair + # process as we are here. + authorized_nodes = + DateTime.utc_now() + |> BeaconChain.previous_summary_time() + |> P2P.authorized_and_available_nodes(true) + + storage_addresses = if storage_address != nil, do: [storage_address], else: [] + + data = %{ + storage_addresses: storage_addresses, + io_addresses: io_addresses, + authorized_nodes: authorized_nodes + } + + {:ok, start_repair(data)} + end + + def handle_cast({:add_address, storage_address, io_addresses}, data) do + new_data = + if storage_address != nil, + do: Map.update!(data, :storage_addresses, &([storage_address | &1] |> Enum.uniq())), + else: data + + new_data = + if io_addresses != [], + do: Map.update!(new_data, :io_addresses, &((&1 ++ io_addresses) |> Enum.uniq())), + else: new_data + + {:noreply, new_data} + end + + def handle_info( + {:DOWN, _ref, :process, pid, _normal}, + data = %{task: task_pid, storage_addresses: [], io_addresses: []} + ) + when pid == task_pid do + {:stop, :normal, data} + end + + def handle_info( + {:DOWN, _ref, :process, pid, _normal}, + data = %{task: task_pid} + ) + when pid == task_pid do + {:noreply, start_repair(data)} + end + + def handle_info(_, data), do: {:noreply, data} + + defp start_repair( + data = %{ + storage_addresses: [], + io_addresses: [address | rest], + authorized_nodes: authorized_nodes + } + ) do + pid = repair_task(address, false, authorized_nodes) + + data + |> Map.put(:io_addresses, rest) + |> Map.put(:task, pid) + end + + defp start_repair( + data = %{ + storage_addresses: [address | rest], + authorized_nodes: authorized_nodes + } + ) do + pid = repair_task(address, true, authorized_nodes) + + data + |> Map.put(:storage_addresses, rest) + |> Map.put(:task, pid) + end + + defp repair_task(address, storage?, authorized_nodes) do + %Task{pid: pid} = + Task.async(fn -> + replicate_transaction(address, storage?, authorized_nodes) + end) + + pid + end + + defp replicate_transaction(address, storage?, authorized_nodes) do + Logger.debug("Notifier RepairWorker start replication, storage? #{storage?}", + address: Base.encode16(address) + ) + + with false <- TransactionChain.transaction_exists?(address), + storage_nodes <- Election.chain_storage_nodes(address, authorized_nodes), + {:ok, tx} <- TransactionChain.fetch_transaction_remotely(address, storage_nodes) do + if storage? do + case Replication.validate_and_store_transaction_chain(tx, true, authorized_nodes) do + :ok -> update_last_address(address, authorized_nodes) + error -> error + end + else + Replication.validate_and_store_transaction(tx, true) + end + else + true -> + Logger.debug("Notifier RepairWorker transaction already exists", + address: Base.encode16(address) + ) + + {:error, reason} -> + Logger.warning( + "Notifier RepairWorker failed to replicate transaction because of #{inspect(reason)}" + ) + end + end + + @doc """ + Request missing transaction addresses from last local address until last chain address + and add them in the DB + """ + def update_last_address(address, authorized_nodes) do + # As the node is storage node of this chain, it needs to know all the addresses of the chain until the last + # So we get the local last address and verify if it's the same as the last address of the chain + # by requesting the nodes which already know the last address + + {last_local_address, _timestamp} = TransactionChain.get_last_address(address) + storage_nodes = Election.storage_nodes(last_local_address, authorized_nodes) + + case TransactionChain.fetch_next_chain_addresses_remotely(last_local_address, storage_nodes) do + {:ok, []} -> + :ok + + {:ok, addresses} -> + genesis_address = TransactionChain.get_genesis_address(address) + + addresses + |> Enum.sort_by(fn {_address, timestamp} -> timestamp end) + |> Enum.each(fn {address, timestamp} -> + TransactionChain.register_last_address(genesis_address, address, timestamp) + end) + + # Stop potential previous smart contract + Contracts.stop_contract(address) + + _ -> + :ok + end + end +end diff --git a/lib/archethic/self_repair/supervisor.ex b/lib/archethic/self_repair/supervisor.ex index bfc0efbbd6..cf6188fc9f 100644 --- a/lib/archethic/self_repair/supervisor.ex +++ b/lib/archethic/self_repair/supervisor.ex @@ -3,9 +3,7 @@ defmodule Archethic.SelfRepair.Supervisor do use Supervisor - alias Archethic.SelfRepair.Notifier alias Archethic.SelfRepair.Scheduler - alias Archethic.Utils def start_link(arg) do @@ -15,7 +13,11 @@ defmodule Archethic.SelfRepair.Supervisor do def init(_arg) do children = [ {Scheduler, Application.get_env(:archethic, Scheduler)}, - Notifier + {DynamicSupervisor, strategy: :one_for_one, name: Archethic.SelfRepair.NotifierSupervisor}, + {Registry, + name: Archethic.SelfRepair.RepairRegistry, + keys: :unique, + partitions: System.schedulers_online()} ] Supervisor.init(Utils.configurable_children(children), strategy: :one_for_one) diff --git a/lib/archethic/self_repair/sync.ex b/lib/archethic/self_repair/sync.ex index ff4bfa495b..bdf8738a7f 100644 --- a/lib/archethic/self_repair/sync.ex +++ b/lib/archethic/self_repair/sync.ex @@ -25,6 +25,8 @@ defmodule Archethic.SelfRepair.Sync do alias Archethic.TransactionChain.TransactionSummary + alias Archethic.SelfRepair + alias Archethic.Utils require Logger @@ -123,13 +125,13 @@ defmodule Archethic.SelfRepair.Sync do defp do_load_missed_transactions(last_sync_date, last_summary_time) do start = System.monotonic_time() - download_nodes = P2P.authorized_and_available_nodes() + download_nodes = P2P.authorized_and_available_nodes(last_summary_time, true) summaries_aggregates = fetch_summaries_aggregates(last_sync_date, last_summary_time, download_nodes) last_aggregate = BeaconChain.fetch_and_aggregate_summaries(last_summary_time, download_nodes) - ensure_download_last_aggregate(last_aggregate) + ensure_download_last_aggregate(last_aggregate, download_nodes) last_aggregate = aggregate_with_local_summaries(last_aggregate, last_summary_time) @@ -166,20 +168,15 @@ defmodule Archethic.SelfRepair.Sync do |> Stream.map(fn {:ok, {:ok, aggregate}} -> aggregate end) end - defp ensure_download_last_aggregate( - last_aggregate = %SummaryAggregate{summary_time: summary_time} - ) do + defp ensure_download_last_aggregate(last_aggregate, download_nodes) do # Make sure the last beacon aggregate have been synchronized # from remote nodes to avoid self-repair to be acknowledged if those # cannot be reached - - nodes = P2P.authorized_and_available_nodes(summary_time) - # If number of authorized node is <= 2 and current node is part of it # we accept the self repair as the other node may be unavailable and so # we need to do the self even if no other node respond with true <- P2P.authorized_node?(), - true <- length(nodes) <= 2 do + true <- length(download_nodes) <= 2 do :ok else _ -> @@ -248,6 +245,8 @@ defmodule Archethic.SelfRepair.Sync do availability_update = DateTime.add(summary_time, availability_adding_time) + previous_available_nodes = P2P.authorized_and_available_nodes() + p2p_availabilities |> Enum.reduce(%{}, fn {subset, %{ @@ -269,9 +268,19 @@ defmodule Archethic.SelfRepair.Sync do |> Enum.map(&update_availabilities(&1, availability_update)) |> DB.register_p2p_summary() + new_available_nodes = P2P.authorized_and_available_nodes(availability_update) + + if :persistent_term.get(:archethic_up, nil) == :up do + SelfRepair.start_notifier( + previous_available_nodes, + new_available_nodes, + availability_update + ) + end + update_statistics(summary_time, transaction_summaries) - store_aggregate(aggregate) + store_aggregate(aggregate, new_available_nodes) end defp synchronize_transactions([], _), do: :ok @@ -378,9 +387,11 @@ defmodule Archethic.SelfRepair.Sync do PubSub.notify_new_tps(tps, nb_transactions) end - defp store_aggregate(aggregate = %SummaryAggregate{summary_time: summary_time}) do - node_list = - [P2P.get_node_info() | P2P.authorized_and_available_nodes()] |> P2P.distinct_nodes() + defp store_aggregate( + aggregate = %SummaryAggregate{summary_time: summary_time}, + new_available_nodes + ) do + node_list = [P2P.get_node_info() | new_available_nodes] |> P2P.distinct_nodes() should_store? = summary_time diff --git a/lib/archethic/transaction_chain.ex b/lib/archethic/transaction_chain.ex index 1742146f8c..ee6ec16b6c 100644 --- a/lib/archethic/transaction_chain.ex +++ b/lib/archethic/transaction_chain.ex @@ -10,25 +10,27 @@ defmodule Archethic.TransactionChain do alias Archethic.Election alias Archethic.P2P - alias Archethic.P2P.Node alias Archethic.P2P.Message + alias Archethic.P2P.Node alias Archethic.P2P.Message.{ + AddressList, Error, - NotFound, - TransactionList, - UnspentOutputList, - TransactionInputList, - TransactionChainLength, - LastTransactionAddress, FirstAddress, - GetTransaction, GetFirstAddress, - GetUnspentOutputs, + GetLastTransactionAddress, + GetNextAddresses, + GetTransaction, GetTransactionChain, + GetTransactionChainLength, GetTransactionInputs, - GetLastTransactionAddress, - GetTransactionChainLength + GetUnspentOutputs, + LastTransactionAddress, + NotFound, + TransactionChainLength, + TransactionInputList, + TransactionList, + UnspentOutputList } alias __MODULE__.MemTables.KOLedger @@ -78,7 +80,7 @@ defmodule Archethic.TransactionChain do @doc """ Stream all the addresses in chronological belonging to a genesis address """ - @spec list_chain_addresses(binary()) :: Enumerable.t() | list({binary(), non_neg_integer()}) + @spec list_chain_addresses(binary()) :: Enumerable.t() | list({binary(), DateTime.t()}) defdelegate list_chain_addresses(genesis_address), to: DB @doc """ @@ -98,10 +100,10 @@ defmodule Archethic.TransactionChain do as: :get_last_chain_address @doc """ - Register a last address from a previous address at a given date + Register a last address from a genesis address at a given date """ @spec register_last_address(binary(), binary(), DateTime.t()) :: :ok - defdelegate register_last_address(previous_address, next_address, timestamp), + defdelegate register_last_address(genesis_address, next_address, timestamp), to: DB, as: :add_last_transaction_address @@ -111,6 +113,16 @@ defmodule Archethic.TransactionChain do @spec get_first_public_key(Crypto.key()) :: Crypto.key() defdelegate get_first_public_key(previous_public_key), to: DB, as: :get_first_public_key + @doc """ + Stream first transactions address of a chain from genesis_address. + The Genesis Addresses is not a transaction or the first transaction. + The first transaction is calulated by index = 0+1 + """ + @spec stream_first_addresses() :: Enumerable.t() + defdelegate stream_first_addresses(), + to: DB, + as: :stream_first_addresses + @doc """ Get a transaction @@ -589,6 +601,29 @@ defmodule Archethic.TransactionChain do end end + @doc """ + Request the chain addresses from paging address to last chain address + """ + @spec fetch_next_chain_addresses_remotely(Crypto.prepended_hash(), list(Node.t())) :: + {:ok, list(Crypto.prepended_hash())} | {:error, :network_issue} + def fetch_next_chain_addresses_remotely(address, nodes) do + conflict_resolver = fn results -> + Enum.sort_by(results, &length(&1.addresses), :desc) |> List.first() + end + + case P2P.quorum_read( + nodes, + %GetNextAddresses{address: address}, + conflict_resolver + ) do + {:ok, %AddressList{addresses: addresses}} -> + {:ok, addresses} + + {:error, :network_issue} = e -> + e + end + end + @doc """ Get a transaction summary from a transaction address """ diff --git a/lib/archethic/utils/regression/benchmarks/end_to_end_validation.ex b/lib/archethic/utils/regression/benchmarks/end_to_end_validation.ex index 67034fe1c2..32ec97dbe6 100644 --- a/lib/archethic/utils/regression/benchmarks/end_to_end_validation.ex +++ b/lib/archethic/utils/regression/benchmarks/end_to_end_validation.ex @@ -16,7 +16,6 @@ defmodule Archethic.Utils.Regression.Benchmark.EndToEndValidation do alias Archethic.TransactionChain.TransactionData.UCOLedger.Transfer, as: UCOTransfer @behaviour Benchmark - def plan([host | _nodes], _opts) do port = Application.get_env(:archethic, ArchethicWeb.Endpoint)[:http][:port] diff --git a/lib/archethic_web/live/chains/beacon_live.ex b/lib/archethic_web/live/chains/beacon_live.ex index a073765a85..373fbb03a3 100644 --- a/lib/archethic_web/live/chains/beacon_live.ex +++ b/lib/archethic_web/live/chains/beacon_live.ex @@ -233,7 +233,9 @@ defmodule ArchethicWeb.BeaconChainLive do defp list_transactions_from_summaries(nil), do: [] defp list_transactions_from_aggregate(date = %DateTime{}) do - case BeaconChain.get_summaries_aggregate(date) do + nodes = P2P.authorized_and_available_nodes() + + case BeaconChain.fetch_summaries_aggregate(date, nodes) do {:ok, %SummaryAggregate{transaction_summaries: tx_summaries}} -> Enum.sort_by(tx_summaries, & &1.timestamp, {:desc, DateTime}) diff --git a/lib/archethic_web/live/chains/node_shared_secrets_live.ex b/lib/archethic_web/live/chains/node_shared_secrets_live.ex index be99f34fc5..7aa9e5b2ca 100644 --- a/lib/archethic_web/live/chains/node_shared_secrets_live.ex +++ b/lib/archethic_web/live/chains/node_shared_secrets_live.ex @@ -153,7 +153,7 @@ defmodule ArchethicWeb.NodeSharedSecretsChainLive do display_data( addr, nb_authorized_nodes, - DateTime.from_unix(timestamp, :millisecond) |> elem(1) + timestamp ) end) |> Enum.reverse() diff --git a/lib/archethic_web/live/chains/reward_live.ex b/lib/archethic_web/live/chains/reward_live.ex index f46942d977..a6b41c1224 100644 --- a/lib/archethic_web/live/chains/reward_live.ex +++ b/lib/archethic_web/live/chains/reward_live.ex @@ -132,7 +132,7 @@ defmodule ArchethicWeb.RewardChainLive do display_data( addr, (TransactionChain.get_transaction(addr, [:type]) |> elem(1)).type, - DateTime.from_unix(timestamp, :millisecond) |> elem(1) + timestamp ) end) |> Enum.reverse() diff --git a/test/archethic/p2p/message/address_list_test.exs b/test/archethic/p2p/message/address_list_test.exs new file mode 100644 index 0000000000..ad2f699cf7 --- /dev/null +++ b/test/archethic/p2p/message/address_list_test.exs @@ -0,0 +1,7 @@ +defmodule Archethic.P2P.Message.AddressListTest do + @moduledoc false + use ExUnit.Case + + alias Archethic.P2P.Message.AddressList + doctest AddressList +end diff --git a/test/archethic/p2p/message/get_next_addresses_test.exs b/test/archethic/p2p/message/get_next_addresses_test.exs new file mode 100644 index 0000000000..9df5e01fae --- /dev/null +++ b/test/archethic/p2p/message/get_next_addresses_test.exs @@ -0,0 +1,7 @@ +defmodule Archethic.P2P.Message.GetNextAddressesTest do + @moduledoc false + use ExUnit.Case + + alias Archethic.P2P.Message.GetNextAddresses + doctest GetNextAddresses +end diff --git a/test/archethic/p2p/message/shard_repair_test.exs b/test/archethic/p2p/message/shard_repair_test.exs new file mode 100644 index 0000000000..3da2a35203 --- /dev/null +++ b/test/archethic/p2p/message/shard_repair_test.exs @@ -0,0 +1,7 @@ +defmodule Archethic.P2P.Message.ShardRepairTest do + @moduledoc false + use ExUnit.Case + + alias Archethic.P2P.Message.ShardRepair + doctest ShardRepair +end diff --git a/test/archethic/p2p/messages_test.exs b/test/archethic/p2p/messages_test.exs index dd3d1ca7a7..5e9fd85366 100644 --- a/test/archethic/p2p/messages_test.exs +++ b/test/archethic/p2p/messages_test.exs @@ -1,64 +1,68 @@ defmodule Archethic.P2P.MessageTest do use ArchethicCase - alias Archethic.Crypto - - alias Archethic.P2P.Message - alias Archethic.P2P.Message.AcknowledgeStorage - alias Archethic.P2P.Message.AddMiningContext - alias Archethic.P2P.Message.Balance - alias Archethic.P2P.Message.BootstrappingNodes - alias Archethic.P2P.Message.CrossValidate - alias Archethic.P2P.Message.CrossValidationDone - alias Archethic.P2P.Message.EncryptedStorageNonce - alias Archethic.P2P.Message.Error - alias Archethic.P2P.Message.FirstPublicKey - alias Archethic.P2P.Message.GetBalance - alias Archethic.P2P.Message.GetBootstrappingNodes - alias Archethic.P2P.Message.GetFirstPublicKey - alias Archethic.P2P.Message.GetLastTransaction - alias Archethic.P2P.Message.GetLastTransactionAddress - alias Archethic.P2P.Message.GetP2PView - alias Archethic.P2P.Message.GetStorageNonce - alias Archethic.P2P.Message.GetTransaction - alias Archethic.P2P.Message.GetTransactionChain - alias Archethic.P2P.Message.GetTransactionChainLength - alias Archethic.P2P.Message.GetTransactionInputs - alias Archethic.P2P.Message.GetTransactionSummary - alias Archethic.P2P.Message.GetUnspentOutputs - alias Archethic.P2P.Message.LastTransactionAddress - alias Archethic.P2P.Message.ListNodes - alias Archethic.P2P.Message.NewTransaction - alias Archethic.P2P.Message.NodeList - alias Archethic.P2P.Message.NotFound - alias Archethic.P2P.Message.NotifyEndOfNodeSync - alias Archethic.P2P.Message.NotifyLastTransactionAddress - alias Archethic.P2P.Message.NotifyPreviousChain - alias Archethic.P2P.Message.Ok - alias Archethic.P2P.Message.P2PView - alias Archethic.P2P.Message.Ping - alias Archethic.P2P.Message.RegisterBeaconUpdates - alias Archethic.P2P.Message.ReplicateTransaction - alias Archethic.P2P.Message.ReplicateTransactionChain - alias Archethic.P2P.Message.StartMining - alias Archethic.P2P.Message.StartMining - alias Archethic.P2P.Message.TransactionChainLength - alias Archethic.P2P.Message.TransactionInputList - alias Archethic.P2P.Message.TransactionList - alias Archethic.P2P.Message.UnspentOutputList - alias Archethic.P2P.Node - - alias Archethic.TransactionChain.Transaction - alias Archethic.TransactionChain.Transaction.CrossValidationStamp - alias Archethic.TransactionChain.Transaction.ValidationStamp - alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations - alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations.UnspentOutput - - alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations.VersionedUnspentOutput - - alias Archethic.TransactionChain.TransactionData - alias Archethic.TransactionChain.TransactionInput - alias Archethic.TransactionChain.VersionedTransactionInput + alias Archethic.{ + Crypto, + P2P.Message, + P2P.Node + } + + alias Archethic.P2P.Message.{ + AcknowledgeStorage, + AddMiningContext, + Balance, + BootstrappingNodes, + CrossValidate, + CrossValidationDone, + EncryptedStorageNonce, + Error, + FirstPublicKey, + GetBalance, + GetBootstrappingNodes, + GetFirstPublicKey, + GetLastTransaction, + GetLastTransactionAddress, + GetP2PView, + GetStorageNonce, + GetTransaction, + GetTransactionChain, + GetTransactionChainLength, + GetTransactionInputs, + GetTransactionSummary, + GetUnspentOutputs, + LastTransactionAddress, + ListNodes, + NewTransaction, + NodeList, + NotFound, + NotifyEndOfNodeSync, + NotifyLastTransactionAddress, + NotifyPreviousChain, + Ok, + P2PView, + Ping, + RegisterBeaconUpdates, + ReplicateTransaction, + ReplicateTransactionChain, + StartMining, + ShardRepair, + TransactionChainLength, + TransactionInputList, + TransactionList, + UnspentOutputList + } + + alias Archethic.TransactionChain.{ + TransactionData, + TransactionInput, + VersionedTransactionInput, + Transaction, + Transaction.CrossValidationStamp, + Transaction.ValidationStamp, + Transaction.ValidationStamp.LedgerOperations, + Transaction.ValidationStamp.LedgerOperations.UnspentOutput, + Transaction.ValidationStamp.LedgerOperations.VersionedUnspentOutput + } doctest Message @@ -980,6 +984,23 @@ defmodule Archethic.P2P.MessageTest do |> Message.decode() |> elem(0) end + + test "%ShardRepair" do + msg = %ShardRepair{ + first_address: <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>>, + storage_address: <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>>, + io_addresses: [ + <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>>, + <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + ] + } + + assert msg == + msg + |> Message.encode() + |> Message.decode() + |> elem(0) + end end test "get_timeout should return timeout according to message type" do diff --git a/test/archethic/replication_test.exs b/test/archethic/replication_test.exs index a62e14be72..fa95916df8 100644 --- a/test/archethic/replication_test.exs +++ b/test/archethic/replication_test.exs @@ -255,7 +255,7 @@ defmodule Archethic.ReplicationTest do |> expect(:get_transaction, fn _, _ -> {:ok, %Transaction{validation_stamp: %ValidationStamp{timestamp: DateTime.utc_now()}}} end) - |> expect(:list_chain_addresses, fn _ -> [{"@Alice1", 0}] end) + |> expect(:list_chain_addresses, fn _ -> [{"@Alice1", DateTime.utc_now()}] end) MockClient |> stub(:send_message, fn _, diff --git a/test/archethic/self_repair/notifier_test.exs b/test/archethic/self_repair/notifier_test.exs index 5b073b8393..9c0b65e3d8 100644 --- a/test/archethic/self_repair/notifier_test.exs +++ b/test/archethic/self_repair/notifier_test.exs @@ -1,88 +1,259 @@ defmodule Archethic.SelfRepair.NotifierTest do + @moduledoc false use ArchethicCase + import Mox + + alias Archethic.BeaconChain.SummaryAggregate + alias Archethic.BeaconChain.SummaryTimer + alias Archethic.Crypto + alias Archethic.Election + alias Archethic.P2P + alias Archethic.P2P.Message.GetBeaconSummariesAggregate alias Archethic.P2P.Message.Ok - alias Archethic.P2P.Message.ReplicateTransaction + alias Archethic.P2P.Message.ShardRepair alias Archethic.P2P.Node alias Archethic.SelfRepair.Notifier + alias Archethic.TransactionChain.Transaction alias Archethic.TransactionChain.Transaction.ValidationStamp + alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations - import Mox + alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations - test "when a node is becoming offline new nodes should receive transaction to replicate" do - P2P.add_and_connect_node(%Node{ - first_public_key: Crypto.first_node_public_key(), - last_public_key: Crypto.first_node_public_key(), - ip: {127, 0, 0, 1}, - port: 3000, + test "new_storage_nodes/2 should return new election" do + node1 = %Node{ + first_public_key: "node1", + last_public_key: "node1", authorized?: true, - authorization_date: ~U[2022-02-01 00:00:00Z], + authorization_date: DateTime.utc_now(), + available?: true, geo_patch: "AAA" - }) + } - P2P.add_and_connect_node(%Node{ + node2 = %Node{ first_public_key: "node2", last_public_key: "node2", - ip: {127, 0, 0, 1}, - port: 3001, authorized?: true, - authorization_date: ~U[2022-02-01 00:00:00Z], - geo_patch: "CCC" - }) + authorization_date: DateTime.utc_now(), + available?: true, + geo_patch: "AAA" + } - P2P.add_and_connect_node(%Node{ + node3 = %Node{ first_public_key: "node3", last_public_key: "node3", - ip: {127, 0, 0, 1}, - port: 3002, authorized?: true, - authorization_date: ~U[2022-02-03 00:00:00Z], - geo_patch: "DDD" - }) + authorization_date: DateTime.utc_now(), + available?: true, + geo_patch: "AAA" + } - {:ok, pid} = Notifier.start_link() + node4 = %Node{ + first_public_key: "node4", + last_public_key: "node4", + authorized?: true, + authorization_date: DateTime.utc_now(), + available?: true, + geo_patch: "AAA" + } - MockDB - |> expect(:list_transactions, fn _ -> - [ - %Transaction{ - address: "@Alice1", - type: :transfer, - validation_stamp: %ValidationStamp{ - timestamp: ~U[2022-02-01 12:54:00Z] - } + P2P.add_and_connect_node(node1) + P2P.add_and_connect_node(node2) + P2P.add_and_connect_node(node3) + P2P.add_and_connect_node(node4) + + prev_storage_nodes = ["node2", "node3"] + new_available_nodes = [node1, node2, node3, node4] + + assert {"Alice1", ["node4", "node1"], []} = + Notifier.new_storage_nodes( + {"Alice1", [], prev_storage_nodes, []}, + new_available_nodes + ) + end + + test "map_last_address_for_node/1 should create a map with last address for each node" do + tab = [ + {"Alice1", ["node1"], ["node3"]}, + {"Alice2", [], ["node1"]}, + {"Alice3", ["node1"], ["node3"]}, + {"Alice4", ["node4"], ["node2"]}, + {"Alice5", ["node3"], []} + ] + + expected = %{ + "node1" => %{last_address: "Alice3", io_addresses: ["Alice2"]}, + "node2" => %{last_address: nil, io_addresses: ["Alice4"]}, + "node3" => %{last_address: "Alice5", io_addresses: ["Alice3", "Alice1"]}, + "node4" => %{last_address: "Alice4", io_addresses: []} + } + + assert ^expected = Notifier.map_last_addresses_for_node(tab) + end + + test "repair_transactions/3 should send message to new storage nodes" do + node = %Node{ + first_public_key: Crypto.first_node_public_key(), + last_public_key: Crypto.last_node_public_key(), + authorized?: true, + authorization_date: DateTime.utc_now(), + available?: true, + geo_patch: "AAA" + } + + P2P.add_and_connect_node(node) + + prev_available_nodes = + Enum.map(1..50, fn nb -> + node = %Node{ + first_public_key: "node#{nb}", + last_public_key: "node#{nb}", + authorized?: true, + authorization_date: DateTime.utc_now(), + available?: true, + geo_patch: "#{Integer.to_string(nb, 16)}A" } - ] + + P2P.add_and_connect_node(node) + + node + end) + + prev_available_nodes = [node | prev_available_nodes] + + # Take nodes in election of Alice2 but not in the one of Alice3 + elec1 = Election.chain_storage_nodes("Alice2", prev_available_nodes) + elec2 = Election.chain_storage_nodes("Alice3", prev_available_nodes) + + diff_nodes = elec1 -- elec2 + + unavailable_nodes = Enum.take(diff_nodes, 2) |> Enum.map(& &1.first_public_key) + + new_available_nodes = + Enum.reject(prev_available_nodes, &(&1.first_public_key in unavailable_nodes)) + + # New possible storage nodes for Alice2 + new_possible_nodes = (prev_available_nodes -- elec1) |> Enum.map(& &1.first_public_key) + + MockDB + |> stub(:stream_first_addresses, fn -> ["Alice1"] end) + |> stub(:get_transaction_chain, fn + "Alice1", _, _ -> + {[ + %Transaction{ + address: "Alice1", + validation_stamp: %ValidationStamp{ + ledger_operations: %LedgerOperations{transaction_movements: []} + } + }, + %Transaction{ + address: "Alice2", + validation_stamp: %ValidationStamp{ + ledger_operations: %LedgerOperations{transaction_movements: []} + } + }, + %Transaction{ + address: "Alice3", + validation_stamp: %ValidationStamp{ + ledger_operations: %LedgerOperations{transaction_movements: []} + } + } + ], false, nil} end) me = self() MockClient - |> expect(:send_message, fn %Node{first_public_key: "node3"}, - %ReplicateTransaction{ - transaction: %Transaction{address: "@Alice1"} - }, - _ -> - send(me, :tx_replicated) - %Ok{} + |> stub(:send_message, fn + node, %ShardRepair{first_address: "Alice1", storage_address: "Alice2"}, _ -> + if Enum.member?(new_possible_nodes, node.first_public_key) do + send(me, :new_node) + end + + _, _, _ -> + {:ok, %Ok{}} end) - send( - pid, - {:node_update, - %Node{ - first_public_key: "node2", - available?: false, - authorized?: true, - authorization_date: ~U[2022-02-01 00:00:00Z] - }} - ) - - assert_receive :tx_replicated + Notifier.repair_transactions(unavailable_nodes, prev_available_nodes, new_available_nodes) + + # Expect to receive only 1 new node for Alice2 + assert_receive :new_node + refute_receive :new_node, 200 + end + + test "repair_summaries_aggregate/2 should store beacon aggregate" do + enrollment_date = DateTime.utc_now() |> DateTime.add(-10, :minute) + + node = %Node{ + first_public_key: Crypto.first_node_public_key(), + last_public_key: Crypto.last_node_public_key(), + geo_patch: "AAA", + network_patch: "AAA", + authorized?: true, + authorization_date: DateTime.utc_now() |> DateTime.add(-11, :minute), + available?: true, + enrollment_date: enrollment_date, + availability_history: <<1::1>> + } + + P2P.add_and_connect_node(node) + + nodes = + Enum.map(1..9, fn nb -> + %Node{ + first_public_key: "node#{nb}", + last_public_key: "node#{nb}", + geo_patch: "AAA", + network_patch: "AAA", + authorized?: true, + authorization_date: DateTime.utc_now(), + available?: true, + availability_history: <<1::1>> + } + end) + + nodes = [node | nodes] + + start_supervised!({SummaryTimer, interval: "0 * * * *"}) + + [first_date | rest] = SummaryTimer.next_summaries(enrollment_date) |> Enum.to_list() + random_date = Enum.random(rest) + + me = self() + + MockDB + |> stub(:get_beacon_summaries_aggregate, fn + summary_time when summary_time in [first_date, random_date] -> + {:error, :not_exists} + + summary_time -> + {:ok, %SummaryAggregate{summary_time: summary_time}} + end) + |> expect(:write_beacon_summaries_aggregate, 2, fn + %SummaryAggregate{summary_time: summary_time} when summary_time == first_date -> + send(me, :write_first_date) + + %SummaryAggregate{summary_time: summary_time} when summary_time == random_date -> + send(me, :write_random_date) + + _ -> + send(me, :unexpected) + end) + + MockClient + |> stub(:send_message, fn _, %GetBeaconSummariesAggregate{date: summary_time}, _ -> + {:ok, %SummaryAggregate{summary_time: summary_time}} + end) + + Notifier.repair_summaries_aggregate(nodes, nodes) + + assert_receive :write_first_date + assert_receive :write_random_date + refute_receive :unexpected end end diff --git a/test/archethic/self_repair/repair_worker_test.exs b/test/archethic/self_repair/repair_worker_test.exs new file mode 100644 index 0000000000..487520ae2d --- /dev/null +++ b/test/archethic/self_repair/repair_worker_test.exs @@ -0,0 +1,164 @@ +defmodule Archethic.SelfRepair.RepairWorkerTest do + @moduledoc false + use ArchethicCase + + alias Archethic.BeaconChain.SummaryTimer + + alias Archethic.Crypto + + alias Archethic.P2P + alias Archethic.P2P.Client.DefaultImpl + alias Archethic.P2P.Node + alias Archethic.P2P.Message.GetNextAddresses + alias Archethic.P2P.Message.GetTransaction + + alias Archethic.SelfRepair.RepairWorker + + alias Archethic.TransactionChain.Transaction + alias Archethic.TransactionChain.Transaction.ValidationStamp + + import Mox + + setup do + start_supervised!({SummaryTimer, interval: "0 0 * * *"}) + + :ok + end + + test "start_link/1 should start a new worker and create a task to replicate transaction" do + {:ok, pid} = + RepairWorker.start_link( + first_address: "Alice1", + storage_address: "Alice2", + io_addresses: ["Bob1"] + ) + + assert %{storage_addresses: [], io_addresses: ["Bob1"], task: _task_pid} = :sys.get_state(pid) + end + + test "repair_task/3 replicate a transaction if it does not already exists" do + P2P.add_and_connect_node(%Node{ + first_public_key: "node1", + last_public_key: "node1", + geo_patch: "AAA", + authorized?: true, + authorization_date: ~U[2022-11-27 00:00:00Z], + available?: true + }) + + {:ok, pid} = + RepairWorker.start_link( + first_address: "Alice1", + storage_address: "Alice2", + io_addresses: ["Bob1", "Bob2"] + ) + + me = self() + + MockDB + |> stub(:transaction_exists?, fn + "Bob2" -> + send(me, :exists_bob3) + true + + _ -> + false + end) + + MockClient + |> stub(:send_message, fn + _, %GetTransaction{address: "Alice2"}, _ -> + send(me, :get_tx_alice2) + + _, %GetTransaction{address: "Bob1"}, _ -> + send(me, :get_tx_bob1) + + _, %GetTransaction{address: "Bob2"}, _ -> + send(me, :get_tx_bob2) + end) + + assert_receive :get_tx_alice2 + assert_receive :get_tx_bob1 + + assert_receive :exists_bob3 + refute_receive :get_tx_bob2 + + assert not Process.alive?(pid) + end + + test "add_message/1 should add new addresses in GenServer state" do + MockDB + |> stub(:transaction_exists?, fn _ -> Process.sleep(100) end) + + {:ok, pid} = + RepairWorker.start_link( + first_address: "Alice1", + storage_address: "Alice2", + io_addresses: ["Bob1", "Bob2"] + ) + + assert %{ + storage_addresses: [], + io_addresses: ["Bob1", "Bob2"], + task: _task_pid + } = :sys.get_state(pid) + + GenServer.cast(pid, {:add_address, "Alice4", ["Bob2", "Bob3"]}) + GenServer.cast(pid, {:add_address, "Alice3", []}) + GenServer.cast(pid, {:add_address, nil, ["Bob4"]}) + + assert %{ + storage_addresses: ["Alice3", "Alice4"], + io_addresses: ["Bob1", "Bob2", "Bob3", "Bob4"], + task: _task_pid + } = :sys.get_state(pid) + end + + test "update_last_address/1 should request missing addresses and add them in DB" do + node = %Node{ + first_public_key: Crypto.first_node_public_key(), + last_public_key: Crypto.last_node_public_key(), + geo_patch: "AAA", + authorized?: true, + authorization_date: ~U[2022-11-27 00:00:00Z], + available?: true, + availability_history: <<1::1>> + } + + me = self() + + MockDB + |> expect(:get_last_chain_address, fn "Alice2" -> {"Alice2", ~U[2022-11-27 00:10:00Z]} end) + |> expect(:get_transaction, fn "Alice2", _ -> + {:ok, %Transaction{validation_stamp: %ValidationStamp{timestamp: ~U[2022-11-27 00:10:00Z]}}} + end) + |> expect(:get_first_chain_address, 2, fn "Alice2" -> "Alice0" end) + |> expect(:list_chain_addresses, fn "Alice0" -> + [ + {"Alice1", ~U[2022-11-27 00:09:00Z]}, + {"Alice2", ~U[2022-11-27 00:10:00Z]}, + {"Alice3", ~U[2022-11-27 00:11:00Z]}, + {"Alice4", ~U[2022-11-27 00:12:00Z]} + ] + end) + |> expect(:add_last_transaction_address, 2, fn + "Alice0", "Alice3", ~U[2022-11-27 00:11:00Z] -> + send(me, :add_alice3) + + "Alice0", "Alice4", ~U[2022-11-27 00:12:00Z] -> + send(me, :add_alice4) + end) + + MockClient + |> expect(:send_message, fn node, msg = %GetNextAddresses{address: "Alice2"}, timeout -> + send(me, :get_next_addresses) + DefaultImpl.send_message(node, msg, timeout) + end) + + RepairWorker.update_last_address("Alice2", [node]) + + assert_receive :get_next_addresses + assert_receive :add_alice3 + assert_receive :add_alice4 + end +end diff --git a/test/archethic_web/live/rewards_live_test.exs b/test/archethic_web/live/rewards_live_test.exs index 639957eb72..4d64158eb1 100644 --- a/test/archethic_web/live/rewards_live_test.exs +++ b/test/archethic_web/live/rewards_live_test.exs @@ -37,7 +37,6 @@ defmodule ArchethicWeb.RewardsLiveTest do time = DateTime.utc_now() |> DateTime.add(3600 * index, :second) - |> DateTime.to_unix() {address, time} end) From 911bb4ee3679e29bf716ff010af2086e3e6dceae Mon Sep 17 00:00:00 2001 From: Neylix Date: Fri, 2 Dec 2022 12:23:45 +0100 Subject: [PATCH 2/9] Notify last address for genesis storage nodes --- lib/archethic/db/embedded_impl/chain_index.ex | 4 ++-- lib/archethic/p2p/message.ex | 5 ++--- lib/archethic/replication.ex | 1 + 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/archethic/db/embedded_impl/chain_index.ex b/lib/archethic/db/embedded_impl/chain_index.ex index 1da38d4d8d..867c706f30 100644 --- a/lib/archethic/db/embedded_impl/chain_index.ex +++ b/lib/archethic/db/embedded_impl/chain_index.ex @@ -407,7 +407,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do unix_time = DateTime.utc_now() |> DateTime.to_unix(:millisecond) search_last_address_until(genesis_address, unix_time, db_path) || - {address, DateTime.utc_now()} + {address, DateTime.from_unix!(0, :millisecond)} [{_, last_address, last_time}] -> {last_address, DateTime.from_unix!(last_time, :millisecond)} @@ -421,7 +421,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do unix_time = DateTime.utc_now() |> DateTime.to_unix(:millisecond) search_last_address_until(address, unix_time, db_path) || - {address, DateTime.utc_now()} + {address, DateTime.from_unix!(0, :millisecond)} [{_, last_address, last_time}] -> {last_address, DateTime.from_unix!(last_time, :millisecond)} diff --git a/lib/archethic/p2p/message.ex b/lib/archethic/p2p/message.ex index 140a309bfb..4d65a69563 100644 --- a/lib/archethic/p2p/message.ex +++ b/lib/archethic/p2p/message.ex @@ -1719,10 +1719,9 @@ defmodule Archethic.P2P.Message do }, _ ) do - with {local_last_address, local_last_timestamp} <- + with {local_last_address, _} <- TransactionChain.get_last_address(genesis_address), - true <- local_last_address != last_address, - :gt <- DateTime.compare(timestamp, local_last_timestamp) do + true <- local_last_address != last_address do TransactionChain.register_last_address(genesis_address, last_address, timestamp) # Stop potential previous smart contract diff --git a/lib/archethic/replication.ex b/lib/archethic/replication.ex index 8ec0bd0100..98f397ea89 100644 --- a/lib/archethic/replication.ex +++ b/lib/archethic/replication.ex @@ -350,6 +350,7 @@ defmodule Archethic.Replication do # Send a message to all the previous storage nodes address |> TransactionChain.list_chain_addresses() + |> Stream.concat([{genesis_address, nil}]) |> Stream.flat_map(fn {address, _} -> Election.chain_storage_nodes(address, P2P.authorized_and_available_nodes()) end) From 58ec61e597740955cc6e76f0ac7df6d25c350c87 Mon Sep 17 00:00:00 2001 From: bchamagne <74045243+bchamagne@users.noreply.github.com> Date: Fri, 2 Dec 2022 13:56:29 +0100 Subject: [PATCH 3/9] Make P2P reconnection asynchronous (#721) --- lib/archethic/p2p/client/connection.ex | 76 ++++++++--- test/archethic/p2p/client/connection_test.exs | 126 ++++++++++++++++++ 2 files changed, 181 insertions(+), 21 deletions(-) diff --git a/lib/archethic/p2p/client/connection.ex b/lib/archethic/p2p/client/connection.ex index 319c0f1675..dfd67d7fb8 100644 --- a/lib/archethic/p2p/client/connection.ex +++ b/lib/archethic/p2p/client/connection.ex @@ -1,5 +1,13 @@ defmodule Archethic.P2P.Client.Connection do - @moduledoc false + @moduledoc """ + + 3 states: + :initializing + {:connected, socket} + :disconnected + + we use the :initializing state to be able to postpone calls and casts until after the 1 connect attempt + """ alias Archethic.Crypto @@ -77,8 +85,17 @@ defmodule Archethic.P2P.Client.Connection do availability_timer: {nil, 0} } - actions = [{:next_event, :internal, :connect}] - {:ok, :disconnected, data, actions} + {:ok, :initializing, data, [{:next_event, :internal, :connect}]} + end + + # every messages sent while inializing will wait until state changes + def handle_event({:call, _}, _action, :initializing, _data) do + {:keep_state_and_data, :postpone} + end + + # every messages sent while inializing will wait until state changes + def handle_event(:cast, _action, :initializing, _data) do + {:keep_state_and_data, :postpone} end def handle_event( @@ -121,8 +138,6 @@ defmodule Archethic.P2P.Client.Connection do end)} end - def handle_event(:enter, :disconnected, :disconnected, _data), do: :keep_state_and_data - def handle_event( :enter, {:connected, _socket}, @@ -161,7 +176,7 @@ defmodule Archethic.P2P.Client.Connection do def handle_event( :enter, - :disconnected, + _, {:connected, _socket}, data = %{node_public_key: node_public_key} ) do @@ -180,37 +195,41 @@ defmodule Archethic.P2P.Client.Connection do {:keep_state, new_data} end + def handle_event(:enter, _old_state, :initializing, _data), do: :keep_state_and_data + def handle_event(:enter, _old_state, :disconnected, _data), do: :keep_state_and_data def handle_event(:enter, _old_state, {:connected, _socket}, _data), do: :keep_state_and_data + # called from the :disconnected or :initializing state def handle_event( :internal, :connect, - :disconnected, - data = %{ + _state, + _data = %{ ip: ip, port: port, transport: transport } ) do - case transport.handle_connect(ip, port) do - {:ok, socket} -> - {:next_state, {:connected, socket}, data} + # try to connect asynchronously so it does not block the messages coming in + # Task.async/1 will send a {:info, {ref, result}} message to the connection process + Task.async(fn -> + transport.handle_connect(ip, port) + end) - {:error, _} -> - actions = [{{:timeout, :reconnect}, 500, nil}] - {:keep_state_and_data, actions} - end - end - - def handle_event({:timeout, :reconnect}, _event_data, :disconnected, _data) do - actions = [{:next_event, :internal, :connect}] - {:keep_state_and_data, actions} + :keep_state_and_data end + # this message is used to delay next connection attempt def handle_event({:timeout, :reconnect}, _event_data, {:connected, _socket}, _data) do :keep_state_and_data end + # this message is used to delay next connection attempt + def handle_event({:timeout, :reconnect}, _event_data, _state, _data) do + actions = [{:next_event, :internal, :connect}] + {:keep_state_and_data, actions} + end + def handle_event( :cast, {:send_message, ref, from, _msg, _timeout}, @@ -364,11 +383,21 @@ defmodule Archethic.P2P.Client.Connection do end end + # Task.async tells us the process ended successfully def handle_event(:info, {:DOWN, _ref, :process, _pid, :normal}, _, _data) do :keep_state_and_data end - def handle_event(:info, _event, :disconnected, _data), do: :keep_state_and_data + # Task.async sending us the result of the handle_connect + def handle_event(:info, {_ref, {:ok, socket}}, _, data) do + {:next_state, {:connected, socket}, data} + end + + # Task.async sending us the result of the handle_connect + def handle_event(:info, {_ref, {:error, _reason}}, _, data) do + actions = [{{:timeout, :reconnect}, 500, nil}] + {:next_state, :disconnected, data, actions} + end def handle_event( :info, @@ -444,5 +473,10 @@ defmodule Archethic.P2P.Client.Connection do end end + def handle_event(:info, _, _, _data) do + # Unhandled message received + :keep_state_and_data + end + def code_change(_old_vsn, state, data, _extra), do: {:ok, state, data} end diff --git a/test/archethic/p2p/client/connection_test.exs b/test/archethic/p2p/client/connection_test.exs index e28de8b66e..4513ae49b2 100644 --- a/test/archethic/p2p/client/connection_test.exs +++ b/test/archethic/p2p/client/connection_test.exs @@ -17,6 +17,10 @@ defmodule Archethic.P2P.Client.ConnectionTest do node_public_key: "key1" ) + assert {:initializing, _} = :sys.get_state(pid) + + Process.sleep(10) + assert {{:connected, _socket}, %{request_id: 0, messages: %{}}} = :sys.get_state(pid) end @@ -45,6 +49,122 @@ defmodule Archethic.P2P.Client.ConnectionTest do }} = :sys.get_state(pid) end + test "should get an error, :closed when trying to reach an unreachable node" do + defmodule MockTransportUnreachable do + alias Archethic.P2P.Client.Transport + + @behaviour Transport + + def handle_connect(_ip, _port) do + {:error, :timeout} + end + + def handle_send(_socket, <<0::32, _rest::bitstring>>), do: :ok + + def handle_message({_, _, _}), do: {:error, :closed} + end + + {:ok, _} = + Connection.start_link( + transport: MockTransportUnreachable, + ip: {127, 0, 0, 2}, + port: 3000, + node_public_key: Crypto.first_node_public_key() + ) + + assert {:error, :closed} = + Connection.send_message( + Crypto.first_node_public_key(), + %GetBalance{address: <<0::8, :crypto.strong_rand_bytes(32)::binary>>} + ) + end + + test "reconnection should be asynchronous" do + defmodule MockTransportConnectionTimeout do + alias Archethic.P2P.Client.Transport + + @behaviour Transport + + def handle_connect({127, 0, 0, 1}, _port) do + {:error, :timeout} + end + + def handle_connect({127, 0, 0, 2}, _port) do + Process.sleep(100_000) + {:error, :timeout} + end + + def handle_send(_socket, <<0::32, _rest::bitstring>>), do: :ok + + def handle_message({_, _, _}), do: {:error, :closed} + end + + {:ok, pid} = + Connection.start_link( + transport: MockTransportConnectionTimeout, + ip: {127, 0, 0, 1}, + port: 3000, + node_public_key: Crypto.first_node_public_key() + ) + + :sys.replace_state(pid, fn {state, data} -> + {state, Map.put(data, :ip, {127, 0, 0, 2})} + end) + + # 500ms to wait for the 1st reconnect attempt + Process.sleep(550) + + time = System.monotonic_time(:millisecond) + + assert {:error, :closed} = + Connection.send_message( + Crypto.first_node_public_key(), + %GetBalance{address: <<0::8, :crypto.strong_rand_bytes(32)::binary>>}, + 200 + ) + + # ensure there was no delay + time2 = System.monotonic_time(:millisecond) + assert time2 - time < 100 + end + + test "should be in :connected state after reconnection" do + defmodule MockTransportReconnectionSuccess do + alias Archethic.P2P.Client.Transport + + @behaviour Transport + + def handle_connect({127, 0, 0, 1}, _port) do + {:error, :timeout} + end + + def handle_connect({127, 0, 0, 2}, _port) do + {:ok, make_ref()} + end + + def handle_send(_socket, <<0::32, _rest::bitstring>>), do: :ok + + def handle_message({_, _, _}), do: {:error, :closed} + end + + {:ok, pid} = + Connection.start_link( + transport: MockTransportReconnectionSuccess, + ip: {127, 0, 0, 1}, + port: 3000, + node_public_key: Crypto.first_node_public_key() + ) + + :sys.replace_state(pid, fn {state, data} -> + {state, Map.put(data, :ip, {127, 0, 0, 2})} + end) + + # 500ms to wait for the 1st reconnect attempt + Process.sleep(550) + + assert {{:connected, _socket}, _} = :sys.get_state(pid) + end + test "should get an error when the timeout is reached" do {:ok, pid} = Connection.start_link( @@ -234,6 +354,9 @@ defmodule Archethic.P2P.Client.ConnectionTest do node_public_key: "key1" ) + assert {:initializing, _} = :sys.get_state(pid) + Process.sleep(10) + assert {{:connected, _socket}, %{availability_timer: {start, 0}}} = :sys.get_state(pid) assert start != nil end @@ -299,6 +422,9 @@ defmodule Archethic.P2P.Client.ConnectionTest do node_public_key: Crypto.first_node_public_key() ) + assert {:initializing, _} = :sys.get_state(pid) + Process.sleep(10) + assert {{:connected, _socket}, %{availability_timer: {start, 0}}} = :sys.get_state(pid) assert start != nil From 962cfee92a370767ddeaa43bf6aec33a79d070b5 Mon Sep 17 00:00:00 2001 From: bchamagne <74045243+bchamagne@users.noreply.github.com> Date: Fri, 2 Dec 2022 13:57:50 +0100 Subject: [PATCH 4/9] Skip replication if there is a cross-validation error (#718) It also includes: * stop the FSM in case of consensus_not_reached * Validation will check for sufficient funds --- lib/archethic/mining/distributed_workflow.ex | 167 +++++----- lib/archethic/mining/validation_context.ex | 168 ++++++---- .../transaction/validation_stamp.ex | 4 +- .../mining/distributed_workflow_test.exs | 302 ++++++++++++++++++ .../mining/validation_context_test.exs | 8 + 5 files changed, 500 insertions(+), 149 deletions(-) diff --git a/lib/archethic/mining/distributed_workflow.ex b/lib/archethic/mining/distributed_workflow.ex index 6f7a9b3d49..1e0864b6ac 100644 --- a/lib/archethic/mining/distributed_workflow.ex +++ b/lib/archethic/mining/distributed_workflow.ex @@ -502,11 +502,8 @@ defmodule Archethic.Mining.DistributedWorkflow do transaction_type: tx.type ) - next_events = [ - {:next_event, :internal, {:notify_error, :timeout}} - ] - - {:keep_state_and_data, next_events} + notify_error(:timeout, data) + :stop _ -> new_context = @@ -594,11 +591,7 @@ defmodule Archethic.Mining.DistributedWorkflow do if ValidationContext.atomic_commitment?(new_context) do {:next_state, :replication, %{data | context: new_context}} else - next_events = [ - {:next_event, :internal, {:notify_error, :consensus_not_reached}} - ] - - {:next_state, :consensus_not_reached, %{data | context: new_context}, next_events} + {:next_state, :consensus_not_reached, %{data | context: new_context}} end else {:keep_state, %{data | context: new_context}} @@ -609,7 +602,7 @@ defmodule Archethic.Mining.DistributedWorkflow do :enter, :wait_cross_validation_stamps, :consensus_not_reached, - _data = %{ + data = %{ context: context = %ValidationContext{ transaction: tx, @@ -635,14 +628,15 @@ defmodule Archethic.Mining.DistributedWorkflow do MaliciousDetection.start_link(context) - :keep_state_and_data + notify_error(:consensus_not_reached, data) + :stop end def handle_event( :enter, from_state, :replication, - _data = %{ + data = %{ context: context = %ValidationContext{ transaction: %Transaction{address: tx_address, type: type} @@ -650,13 +644,22 @@ defmodule Archethic.Mining.DistributedWorkflow do } ) when from_state in [:cross_validator, :wait_cross_validation_stamps] do - Logger.info("Start replication", - transaction_address: Base.encode16(tx_address), - transaction_type: type - ) + case ValidationContext.get_first_error(context) do + nil -> + Logger.info("Start replication", + transaction_address: Base.encode16(tx_address), + transaction_type: type + ) - request_replication(context) - :keep_state_and_data + request_replication(context) + :keep_state_and_data + + err -> + Logger.info("Skipped replication because validation failed", err: err) + + notify_error(err, data) + :stop + end end def handle_event( @@ -766,19 +769,15 @@ defmodule Archethic.Mining.DistributedWorkflow do :info, {:replication_error, reason}, :replication, - _data = %{context: %ValidationContext{transaction: tx}} + data = %{context: %ValidationContext{transaction: tx}} ) do Logger.error("Replication error - #{inspect(reason)}", transaction_address: Base.encode16(tx.address), transaction_type: tx.type ) - next_events = [ - {:next_event, :internal, {:notify_error, reason}} - ] - - {:keep_state_and_data, next_events} - # :stop + notify_error(reason, data) + :stop end def handle_event( @@ -803,13 +802,8 @@ defmodule Archethic.Mining.DistributedWorkflow do transaction_type: tx.type ) - next_events = [ - {:next_event, :internal, {:notify_error, :timeout}} - ] - - {:keep_state_and_data, next_events} - - # :stop + notify_error(:timeout, data) + :stop _ -> nb_cross_validation_nodes = length(next_cross_validation_nodes) @@ -837,71 +831,14 @@ defmodule Archethic.Mining.DistributedWorkflow do {:timeout, :stop_timeout}, :any, _state, - _data = %{context: %ValidationContext{transaction: tx}} + data = %{context: %ValidationContext{transaction: tx}} ) do Logger.warning("Timeout reached during mining", transaction_type: tx.type, transaction_address: Base.encode16(tx.address) ) - next_events = [ - {:next_event, :internal, {:notify_error, :timeout}} - ] - - {:keep_state_and_data, next_events} - end - - def handle_event( - :internal, - {:notify_error, reason}, - _, - _data = %{ - context: - _context = %ValidationContext{ - welcome_node: welcome_node = %Node{}, - transaction: %Transaction{address: tx_address}, - pending_transaction_error_detail: pending_error_detail - } - } - ) do - {error_context, error_reason} = - case reason do - :invalid_pending_transaction -> - {:invalid_transaction, pending_error_detail} - - :invalid_inherit_constraints -> - {:invalid_transaction, "Inherit constraints not respected"} - - :insufficient_funds -> - {:invalid_transaction, "Insufficient funds"} - - :invalid_proof_of_work -> - {:invalid_transaction, "Invalid origin signature"} - - reason -> - {:network_issue, reason |> Atom.to_string() |> String.replace("_", " ")} - end - - Logger.warning("Invalid transaction #{inspect(error_reason)}", - transaction_address: Base.encode16(tx_address) - ) - - Logger.debug("Notify error back to the welcome node", - transaction_address: Base.encode16(tx_address) - ) - - # Notify error to the welcome node - message = %ValidationError{context: error_context, reason: error_reason, address: tx_address} - - Task.Supervisor.async_nolink(Archethic.TaskSupervisor, fn -> - P2P.send_message( - welcome_node, - message - ) - - :ok - end) - + notify_error(:timeout, data) :stop end @@ -1024,4 +961,50 @@ defmodule Archethic.Mining.DistributedWorkflow do P2P.broadcast_message(storage_nodes, message) end + + defp notify_error(reason, %{ + context: %ValidationContext{ + welcome_node: welcome_node = %Node{}, + transaction: %Transaction{address: tx_address}, + pending_transaction_error_detail: pending_error_detail + } + }) do + {error_context, error_reason} = + case reason do + :invalid_pending_transaction -> + {:invalid_transaction, pending_error_detail} + + :invalid_inherit_constraints -> + {:invalid_transaction, "Inherit constraints not respected"} + + :insufficient_funds -> + {:invalid_transaction, "Insufficient funds"} + + :invalid_proof_of_work -> + {:invalid_transaction, "Invalid origin signature"} + + reason -> + {:network_issue, reason |> Atom.to_string() |> String.replace("_", " ")} + end + + Logger.warning("Invalid transaction #{inspect(error_reason)}", + transaction_address: Base.encode16(tx_address) + ) + + Logger.debug("Notify error back to the welcome node", + transaction_address: Base.encode16(tx_address) + ) + + # Notify error to the welcome node + message = %ValidationError{context: error_context, reason: error_reason, address: tx_address} + + Task.Supervisor.async_nolink(Archethic.TaskSupervisor, fn -> + P2P.send_message( + welcome_node, + message + ) + + :ok + end) + end end diff --git a/lib/archethic/mining/validation_context.ex b/lib/archethic/mining/validation_context.ex index 35bbac4993..d8aafeeec7 100644 --- a/lib/archethic/mining/validation_context.ex +++ b/lib/archethic/mining/validation_context.ex @@ -717,6 +717,46 @@ defmodule Archethic.Mining.ValidationContext do resolved_addresses: resolved_addresses } ) do + resolved_recipients = + Enum.reduce(resolved_addresses, [], fn {to, resolved}, acc -> + if to in recipients do + [resolved | acc] + else + acc + end + end) + + ledger_operations = get_ledger_operations(context) + + validation_stamp = + %ValidationStamp{ + protocol_version: Mining.protocol_version(), + timestamp: validation_time, + proof_of_work: do_proof_of_work(tx), + proof_of_integrity: TransactionChain.proof_of_integrity([tx, prev_tx]), + proof_of_election: Election.validation_nodes_election_seed_sorting(tx, validation_time), + ledger_operations: ledger_operations, + recipients: resolved_recipients, + error: + get_validation_error( + prev_tx, + tx, + ledger_operations, + unspent_outputs, + valid_pending_transaction? + ) + } + |> ValidationStamp.sign() + + %{context | validation_stamp: validation_stamp} + end + + defp get_ledger_operations(%__MODULE__{ + transaction: tx, + unspent_outputs: unspent_outputs, + validation_time: validation_time, + resolved_addresses: resolved_addresses + }) do usd_price = validation_time |> OracleChain.get_uco_price() @@ -750,51 +790,49 @@ defmodule Archethic.Mining.ValidationContext do acc end) - resolved_recipients = - Enum.reduce(resolved_addresses, [], fn {to, resolved}, acc -> - if to in recipients do - [resolved | acc] - else - acc - end - end) + %LedgerOperations{ + fee: fee, + transaction_movements: resolved_movements + } + |> LedgerOperations.from_transaction(tx, validation_time) + |> LedgerOperations.consume_inputs( + tx.address, + unspent_outputs, + validation_time |> DateTime.truncate(:millisecond) + ) + end - error = - cond do - chain_error?(prev_tx, tx) -> - :invalid_inherit_constraints + @spec get_validation_error( + nil | Transaction.t(), + Transaction.t(), + LedgerOperations.t(), + list(UnspentOutput.t()), + boolean() + ) :: nil | ValidationStamp.error() + defp get_validation_error( + prev_tx, + tx, + ledger_operations, + unspent_outputs, + valid_pending_transaction? + ) do + cond do + chain_error?(prev_tx, tx) -> + :invalid_inherit_constraints - valid_pending_transaction? -> - nil + has_insufficient_funds?(ledger_operations, unspent_outputs) -> + :insufficient_funds - true -> - :invalid_pending_transaction - end + not valid_pending_transaction? -> + :invalid_pending_transaction - validation_stamp = - %ValidationStamp{ - protocol_version: Mining.protocol_version(), - timestamp: validation_time, - proof_of_work: do_proof_of_work(tx), - proof_of_integrity: TransactionChain.proof_of_integrity([tx, prev_tx]), - proof_of_election: Election.validation_nodes_election_seed_sorting(tx, validation_time), - ledger_operations: - %LedgerOperations{ - fee: fee, - transaction_movements: resolved_movements - } - |> LedgerOperations.from_transaction(tx, validation_time) - |> LedgerOperations.consume_inputs( - tx.address, - unspent_outputs, - validation_time |> DateTime.truncate(:millisecond) - ), - recipients: resolved_recipients, - error: error - } - |> ValidationStamp.sign() + true -> + nil + end + end - %{context | validation_stamp: validation_stamp} + defp has_insufficient_funds?(ledger_ops, inputs) do + not LedgerOperations.sufficient_funds?(ledger_ops, inputs) end defp chain_error?(nil, _tx = %Transaction{}), do: false @@ -1055,24 +1093,25 @@ defmodule Archethic.Mining.ValidationContext do ) == fee end - defp valid_stamp_error?(stamp = %ValidationStamp{error: error}, %__MODULE__{ - transaction: tx, - previous_transaction: prev_tx, - valid_pending_transaction?: valid_pending_transaction? - }) do - validated_tx = %{tx | validation_stamp: stamp} + defp valid_stamp_error?( + stamp = %ValidationStamp{error: error}, + context = %__MODULE__{ + transaction: tx, + previous_transaction: prev_tx, + valid_pending_transaction?: valid_pending_transaction?, + unspent_outputs: unspent_outputs + } + ) do + validated_context = %{context | transaction: %{tx | validation_stamp: stamp}} expected_error = - cond do - chain_error?(prev_tx, validated_tx) -> - :invalid_inherit_constraints - - valid_pending_transaction? -> - nil - - true -> - :invalid_pending_transaction - end + get_validation_error( + prev_tx, + tx, + get_ledger_operations(validated_context), + unspent_outputs, + valid_pending_transaction? + ) error == expected_error end @@ -1261,4 +1300,21 @@ defmodule Archethic.Mining.ValidationContext do |> Enum.map(&Enum.at(storage_nodes, &1)) |> Enum.reject(&Utils.key_in_node_list?(chain_storage_nodes, &1.first_public_key)) end + + @doc """ + Get the first available error or nil + """ + @spec get_first_error(t()) :: atom() + def get_first_error(%__MODULE__{ + validation_stamp: %ValidationStamp{error: nil}, + cross_validation_stamps: cross_validation_stamps + }) do + cross_validation_stamps + |> Enum.reduce_while(nil, fn + %CrossValidationStamp{inconsistencies: []}, nil -> {:cont, nil} + %CrossValidationStamp{inconsistencies: [first_error | _rest]}, nil -> {:halt, first_error} + end) + end + + def get_first_error(%__MODULE__{validation_stamp: %ValidationStamp{error: error}}), do: error end diff --git a/lib/archethic/transaction_chain/transaction/validation_stamp.ex b/lib/archethic/transaction_chain/transaction/validation_stamp.ex index e43cf16224..1bc9fc4b40 100755 --- a/lib/archethic/transaction_chain/transaction/validation_stamp.ex +++ b/lib/archethic/transaction_chain/transaction/validation_stamp.ex @@ -21,7 +21,7 @@ defmodule Archethic.TransactionChain.Transaction.ValidationStamp do error: nil ] - @type error :: :invalid_pending_transaction | :invalid_inherit_constraints + @type error :: :invalid_pending_transaction | :invalid_inherit_constraints | :insufficient_funds @typedoc """ Validation performed by a coordinator: @@ -426,8 +426,10 @@ defmodule Archethic.TransactionChain.Transaction.ValidationStamp do defp serialize_error(nil), do: 0 defp serialize_error(:invalid_pending_transaction), do: 1 defp serialize_error(:invalid_inherit_constraints), do: 2 + defp serialize_error(:insufficient_funds), do: 3 defp deserialize_error(0), do: nil defp deserialize_error(1), do: :invalid_pending_transaction defp deserialize_error(2), do: :invalid_inherit_constraints + defp deserialize_error(3), do: :insufficient_funds end diff --git a/test/archethic/mining/distributed_workflow_test.exs b/test/archethic/mining/distributed_workflow_test.exs index cda4712cc8..7803bb0039 100644 --- a/test/archethic/mining/distributed_workflow_test.exs +++ b/test/archethic/mining/distributed_workflow_test.exs @@ -5,6 +5,9 @@ defmodule Archethic.Mining.DistributedWorkflowTest do alias Archethic.Crypto + @publickey1 Crypto.generate_deterministic_keypair("seed2") + @publickey2 Crypto.generate_deterministic_keypair("seed3") + alias Archethic.BeaconChain alias Archethic.BeaconChain.ReplicationAttestation alias Archethic.BeaconChain.SlotTimer, as: BeaconSlotTimer @@ -13,6 +16,7 @@ defmodule Archethic.Mining.DistributedWorkflowTest do alias Archethic.Election alias Archethic.Mining.DistributedWorkflow, as: Workflow + alias Archethic.Mining.Fee alias Archethic.Mining.ValidationContext alias Archethic.P2P @@ -20,17 +24,22 @@ defmodule Archethic.Mining.DistributedWorkflowTest do alias Archethic.P2P.Message.CrossValidate alias Archethic.P2P.Message.CrossValidationDone alias Archethic.P2P.Message.GetTransaction + alias Archethic.P2P.Message.GetTransactionSummary alias Archethic.P2P.Message.GetUnspentOutputs alias Archethic.P2P.Message.NotFound alias Archethic.P2P.Message.Ok alias Archethic.P2P.Message.Ping alias Archethic.P2P.Message.ReplicateTransactionChain alias Archethic.P2P.Message.UnspentOutputList + alias Archethic.P2P.Message.ValidationError alias Archethic.P2P.Node + alias Archethic.TransactionChain alias Archethic.TransactionChain.Transaction alias Archethic.TransactionChain.Transaction.CrossValidationStamp alias Archethic.TransactionChain.Transaction.ValidationStamp + alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations + alias Archethic.TransactionChain.Transaction.ValidationStamp.LedgerOperations.UnspentOutput alias Archethic.TransactionChain.TransactionData alias Archethic.TransactionChain.TransactionSummary @@ -118,6 +127,9 @@ defmodule Archethic.Mining.DistributedWorkflowTest do _, %Ping{}, _ -> {:ok, %Ok{}} + _, %GetTransactionSummary{}, _ -> + {:ok, %NotFound{}} + _, %GetUnspentOutputs{}, _ -> {:ok, %UnspentOutputList{unspent_outputs: []}} @@ -198,6 +210,9 @@ defmodule Archethic.Mining.DistributedWorkflowTest do _, %Ping{}, _ -> {:ok, %Ok{}} + _, %GetTransactionSummary{}, _ -> + {:ok, %NotFound{}} + _, %GetUnspentOutputs{}, _ -> {:ok, %UnspentOutputList{unspent_outputs: []}} @@ -300,6 +315,9 @@ defmodule Archethic.Mining.DistributedWorkflowTest do _, %Ping{}, _ -> {:ok, %Ok{}} + _, %GetTransactionSummary{}, _ -> + {:ok, %NotFound{}} + _, %GetUnspentOutputs{}, _ -> {:ok, %UnspentOutputList{unspent_outputs: []}} @@ -423,6 +441,9 @@ defmodule Archethic.Mining.DistributedWorkflowTest do _, %Ping{}, _ -> {:ok, %Ok{}} + _, %GetTransactionSummary{}, _ -> + {:ok, %NotFound{}} + _, %GetUnspentOutputs{}, _ -> {:ok, %UnspentOutputList{unspent_outputs: []}} @@ -552,6 +573,9 @@ defmodule Archethic.Mining.DistributedWorkflowTest do _, %Ping{}, _ -> {:ok, %Ok{}} + _, %GetTransactionSummary{}, _ -> + {:ok, %NotFound{}} + _, %GetUnspentOutputs{}, _ -> {:ok, %UnspentOutputList{unspent_outputs: []}} @@ -783,6 +807,12 @@ defmodule Archethic.Mining.DistributedWorkflowTest do MockClient |> stub(:send_message, fn + _, %ValidationError{}, _ -> + {:ok, %Ok{}} + + _, %GetTransactionSummary{}, _ -> + {:ok, %NotFound{}} + _, %Ping{}, _ -> {:ok, %Ok{}} @@ -968,7 +998,279 @@ defmodule Archethic.Mining.DistributedWorkflowTest do end assert_receive :replication_done + refute_receive :validation_error end end + + test "should not replicate if there is a validation error", %{tx: tx} do + validation_context = create_context(tx) + + validation_stamp = create_validation_stamp(validation_context) + validation_stamp = %ValidationStamp{validation_stamp | error: :invalid_pending_transaction} + + context = + validation_context + |> ValidationContext.add_validation_stamp(validation_stamp) + + me = self() + + MockClient + |> stub(:send_message, fn + _, %Ping{}, _ -> + {:ok, %Ok{}} + + _, %GetUnspentOutputs{}, _ -> + {:ok, %UnspentOutputList{unspent_outputs: []}} + + _, %ValidationError{}, _ -> + send(me, :validation_error) + {:ok, %Ok{}} + + _, %GetTransactionSummary{}, _ -> + {:ok, %NotFound{}} + + _, %GetTransaction{}, _ -> + {:ok, %Transaction{}} + end) + + {:ok, coordinator_pid} = + Workflow.start_link( + transaction: context.transaction, + welcome_node: context.welcome_node, + validation_nodes: [context.coordinator_node | context.cross_validation_nodes], + node_public_key: context.coordinator_node.last_public_key + ) + + :sys.replace_state(coordinator_pid, fn {:coordinator, %{context: _}} -> + {:wait_cross_validation_stamps, %{context: context}} + end) + + Workflow.add_cross_validation_stamp( + coordinator_pid, + %CrossValidationStamp{ + signature: + Crypto.sign( + [ValidationStamp.serialize(context.validation_stamp), <<>>], + elem(@publickey1, 1) + ), + node_public_key: elem(@publickey1, 0), + inconsistencies: [] + } + ) + + Workflow.add_cross_validation_stamp( + coordinator_pid, + %CrossValidationStamp{ + signature: + Crypto.sign( + [ValidationStamp.serialize(context.validation_stamp), <<>>], + elem(@publickey2, 1) + ), + node_public_key: elem(@publickey2, 0), + inconsistencies: [] + } + ) + + assert_receive :validation_error + refute_receive :ack_replication + refute_receive :replication_done + refute Process.alive?(coordinator_pid) + end + + test "should not replicate if there is a cross validation error", %{tx: tx} do + validation_context = create_context(tx) + + context = + validation_context + |> ValidationContext.add_validation_stamp(create_validation_stamp(validation_context)) + + me = self() + + MockClient + |> stub(:send_message, fn + _, %Ping{}, _ -> + {:ok, %Ok{}} + + _, %GetUnspentOutputs{}, _ -> + {:ok, %UnspentOutputList{unspent_outputs: []}} + + _, %ValidationError{}, _ -> + send(me, :validation_error) + {:ok, %Ok{}} + + _, %GetTransactionSummary{}, _ -> + {:ok, %NotFound{}} + + _, %GetTransaction{}, _ -> + {:ok, %Transaction{}} + end) + + {:ok, coordinator_pid} = + Workflow.start_link( + transaction: context.transaction, + welcome_node: context.welcome_node, + validation_nodes: [context.coordinator_node | context.cross_validation_nodes], + node_public_key: context.coordinator_node.last_public_key + ) + + :sys.replace_state(coordinator_pid, fn {:coordinator, %{context: _}} -> + {:wait_cross_validation_stamps, %{context: context}} + end) + + Workflow.add_cross_validation_stamp( + coordinator_pid, + %CrossValidationStamp{ + signature: + Crypto.sign( + [ValidationStamp.serialize(context.validation_stamp), <<1>>], + elem(@publickey1, 1) + ), + node_public_key: elem(@publickey1, 0), + inconsistencies: [:signature] + } + ) + + Workflow.add_cross_validation_stamp( + coordinator_pid, + %CrossValidationStamp{ + signature: + Crypto.sign( + [ValidationStamp.serialize(context.validation_stamp), <<1>>], + elem(@publickey2, 1) + ), + node_public_key: elem(@publickey2, 0), + inconsistencies: [:signature] + } + ) + + assert_receive :validation_error + refute_receive :ack_replication + refute_receive :replication_done + refute Process.alive?(coordinator_pid) + end + end + + defp create_context( + tx, + validation_time \\ DateTime.utc_now() |> DateTime.truncate(:millisecond) + ) do + {pub1, _} = Crypto.generate_deterministic_keypair("seed") + + welcome_node = %Node{ + last_public_key: pub1, + first_public_key: pub1, + geo_patch: "AAA", + network_patch: "AAA", + ip: {127, 0, 0, 1}, + port: 3000, + reward_address: :crypto.strong_rand_bytes(32), + authorized?: true, + authorization_date: DateTime.utc_now() |> DateTime.add(-2) + } + + coordinator_node = %Node{ + first_public_key: Crypto.first_node_public_key(), + last_public_key: Crypto.last_node_public_key(), + geo_patch: "AAA", + network_patch: "AAA", + ip: {127, 0, 0, 1}, + port: 3000, + reward_address: :crypto.strong_rand_bytes(32), + authorized?: true, + authorization_date: DateTime.utc_now() |> DateTime.add(-2) + } + + cross_validation_nodes = [ + %Node{ + first_public_key: elem(@publickey1, 0), + last_public_key: elem(@publickey1, 0), + geo_patch: "AAA", + network_patch: "AAA", + ip: {127, 0, 0, 1}, + port: 3000, + reward_address: :crypto.strong_rand_bytes(32), + authorized?: true, + authorization_date: DateTime.utc_now() |> DateTime.add(-2) + }, + %Node{ + first_public_key: elem(@publickey2, 0), + last_public_key: elem(@publickey2, 0), + geo_patch: "AAA", + network_patch: "AAA", + ip: {127, 0, 0, 1}, + port: 3000, + reward_address: :crypto.strong_rand_bytes(32), + authorized?: true, + authorization_date: DateTime.utc_now() |> DateTime.add(-2) + } + ] + + previous_storage_nodes = [ + %Node{ + last_public_key: "key2", + first_public_key: "key2", + geo_patch: "AAA", + network_patch: "AAA", + available?: true, + reward_address: :crypto.strong_rand_bytes(32), + authorized?: true, + authorization_date: DateTime.utc_now() |> DateTime.add(-2) + }, + %Node{ + last_public_key: "key3", + first_public_key: "key3", + geo_patch: "DEA", + network_patch: "DEA", + available?: true, + reward_address: :crypto.strong_rand_bytes(32), + authorized?: true, + authorization_date: DateTime.utc_now() |> DateTime.add(-2) + } + ] + + P2P.add_and_connect_node(welcome_node) + P2P.add_and_connect_node(coordinator_node) + Enum.each(cross_validation_nodes, &P2P.add_and_connect_node(&1)) + Enum.each(previous_storage_nodes, &P2P.add_and_connect_node(&1)) + + %ValidationContext{ + transaction: tx, + previous_storage_nodes: previous_storage_nodes, + unspent_outputs: [ + %UnspentOutput{ + from: "@Alice2", + amount: 204_000_000, + type: :UCO, + timestamp: validation_time + } + ], + welcome_node: welcome_node, + coordinator_node: coordinator_node, + cross_validation_nodes: cross_validation_nodes, + cross_validation_nodes_confirmation: <<1::1, 1::1>>, + valid_pending_transaction?: true, + validation_time: validation_time + } + end + + defp create_validation_stamp(%ValidationContext{ + transaction: tx, + unspent_outputs: unspent_outputs, + validation_time: timestamp + }) do + %ValidationStamp{ + timestamp: timestamp, + proof_of_work: Crypto.origin_node_public_key(), + proof_of_integrity: TransactionChain.proof_of_integrity([tx]), + proof_of_election: Election.validation_nodes_election_seed_sorting(tx, DateTime.utc_now()), + ledger_operations: + %LedgerOperations{ + fee: Fee.calculate(tx, 0.07, timestamp), + transaction_movements: Transaction.get_movements(tx) + } + |> LedgerOperations.from_transaction(tx, timestamp) + |> LedgerOperations.consume_inputs(tx.address, unspent_outputs, timestamp), + protocol_version: ArchethicCase.current_protocol_version() + } end end diff --git a/test/archethic/mining/validation_context_test.exs b/test/archethic/mining/validation_context_test.exs index 5c9b71b195..ac72e49898 100644 --- a/test/archethic/mining/validation_context_test.exs +++ b/test/archethic/mining/validation_context_test.exs @@ -41,6 +41,14 @@ defmodule Archethic.Mining.ValidationContextTest do |> ValidationContext.cross_validate() end + test "should get inconsistency when the user has not enough funds" do + validation_context = + %ValidationContext{create_context() | unspent_outputs: []} + |> ValidationContext.create_validation_stamp() + + assert validation_context.validation_stamp.error == :insufficient_funds + end + test "should get inconsistency when the validation stamp signature is invalid" do validation_context = create_context() From 978005907948988c809e139fbc8cee50f698ed68 Mon Sep 17 00:00:00 2001 From: Samuel Manzanera Date: Fri, 2 Dec 2022 16:09:34 +0100 Subject: [PATCH 5/9] Fix asynchronous connection process exit Because we use Task.async, the socket is controlled by default by the current process of the exeecution. But when the process exit the socket is closed. Hence we have to indicate the controlling process of the socket --- lib/archethic/p2p/client/connection.ex | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/archethic/p2p/client/connection.ex b/lib/archethic/p2p/client/connection.ex index dfd67d7fb8..1997930b10 100644 --- a/lib/archethic/p2p/client/connection.ex +++ b/lib/archethic/p2p/client/connection.ex @@ -212,8 +212,19 @@ defmodule Archethic.P2P.Client.Connection do ) do # try to connect asynchronously so it does not block the messages coming in # Task.async/1 will send a {:info, {ref, result}} message to the connection process + me = self() Task.async(fn -> - transport.handle_connect(ip, port) + case transport.handle_connect(ip, port) do + {:ok, socket} when is_port(socket) -> + :gen_tcp.controlling_process(socket, me) + {:ok, socket} + + # only used for tests (make_ref()) + {:ok, socket} -> + {:ok, socket} + {:error, _} = e -> + e + end end) :keep_state_and_data From ce75678faf769b69652f33fe72ba0414e191346a Mon Sep 17 00:00:00 2001 From: Neylix Date: Fri, 2 Dec 2022 16:56:55 +0100 Subject: [PATCH 6/9] Fix validation node number --- lib/archethic/election/constraints/validation.ex | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/archethic/election/constraints/validation.ex b/lib/archethic/election/constraints/validation.ex index 5fb396bdf0..2aa25c07b7 100755 --- a/lib/archethic/election/constraints/validation.ex +++ b/lib/archethic/election/constraints/validation.ex @@ -78,10 +78,15 @@ defmodule Archethic.Election.ValidationConstraints do @default_min_validations + (:math.log10(total_transfers / 100_000_000) |> trunc()) - if validation_number > nb_authorized_nodes do - nb_authorized_nodes - else - validation_number + cond do + validation_number > nb_authorized_nodes -> + nb_authorized_nodes + + validation_number < @default_min_validations -> + min_validation_nodes(nb_authorized_nodes) + + true -> + validation_number end else min_validation_nodes(nb_authorized_nodes) From 64ba6b3e0c332c42c7ba3d0909fc408f1d80798a Mon Sep 17 00:00:00 2001 From: Neylix Date: Fri, 2 Dec 2022 16:58:56 +0100 Subject: [PATCH 7/9] Separate chain and io transaction in DB storage (#738) * Write IO transaction in a new folder to separate logic * Load IO transactions on node start * Fix contract loader on start --- lib/archethic.ex | 4 +- lib/archethic/account.ex | 4 +- lib/archethic/account/mem_tables_loader.ex | 47 ++++++++------ lib/archethic/contracts/loader.ex | 22 ++++--- lib/archethic/db.ex | 7 ++- lib/archethic/db/embedded_impl.ex | 56 ++++++++++++----- lib/archethic/db/embedded_impl/chain_index.ex | 11 +++- .../db/embedded_impl/chain_reader.ex | 54 ++++++++++++++++ .../db/embedded_impl/chain_writer.ex | 52 ++++++++++++++++ lib/archethic/oracle_chain/scheduler.ex | 8 +-- lib/archethic/replication.ex | 14 ++--- lib/archethic/self_repair/sync.ex | 2 +- lib/archethic/transaction_chain.ex | 17 ++++-- .../utils/detect_node_responsiveness.ex | 6 +- .../account/mem_tables_loader_test.exs | 34 ++++++++--- .../archethic/bootstrap/network_init_test.exs | 6 +- test/archethic/bootstrap_test.exs | 10 +-- test/archethic/db/embedded_impl_test.exs | 61 +++++++++++++++++-- test/archethic/replication_test.exs | 4 +- test/archethic/reward_test.exs | 4 +- .../self_repair/repair_worker_test.exs | 6 +- .../sync/transaction_handler_test.exs | 2 +- test/archethic/self_repair/sync_test.exs | 4 +- .../utils/detect_node_responsiveness_test.exs | 12 ++-- test/support/template.ex | 4 +- 25 files changed, 341 insertions(+), 110 deletions(-) diff --git a/lib/archethic.ex b/lib/archethic.ex index 6ece12b7b4..bac0f5f8d7 100644 --- a/lib/archethic.ex +++ b/lib/archethic.ex @@ -11,8 +11,6 @@ defmodule Archethic do alias __MODULE__.P2P alias __MODULE__.P2P.Node - alias __MODULE__.DB - alias __MODULE__.P2P.Message.Balance alias __MODULE__.P2P.Message.GetBalance alias __MODULE__.P2P.Message.NewTransaction @@ -275,7 +273,7 @@ defmodule Archethic do try do {local_chain, paging_address} = with true <- paging_address != nil, - true <- DB.transaction_exists?(paging_address), + true <- TransactionChain.transaction_exists?(paging_address), last_address when last_address != nil <- TransactionChain.get_last_local_address(address), true <- last_address != paging_address do diff --git a/lib/archethic/account.ex b/lib/archethic/account.ex index 780a9e27d6..b1b4690a63 100644 --- a/lib/archethic/account.ex +++ b/lib/archethic/account.ex @@ -60,6 +60,6 @@ defmodule Archethic.Account do @doc """ Load the transaction into the Account context filling the memory tables for ledgers """ - @spec load_transaction(Transaction.t()) :: :ok - defdelegate load_transaction(transaction), to: MemTablesLoader + @spec load_transaction(Transaction.t(), boolean()) :: :ok + defdelegate load_transaction(transaction, io_transaction?), to: MemTablesLoader end diff --git a/lib/archethic/account/mem_tables_loader.ex b/lib/archethic/account/mem_tables_loader.ex index 69e31e1e9b..282da484d1 100644 --- a/lib/archethic/account/mem_tables_loader.ex +++ b/lib/archethic/account/mem_tables_loader.ex @@ -56,9 +56,13 @@ defmodule Archethic.Account.MemTablesLoader do @spec init(args :: list()) :: {:ok, []} def init(_args) do + TransactionChain.list_io_transactions(@query_fields) + |> Stream.each(&load_transaction(&1, true)) + |> Stream.run() + TransactionChain.list_all(@query_fields) |> Stream.reject(&(&1.type in @excluded_types)) - |> Stream.each(&load_transaction/1) + |> Stream.each(&load_transaction(&1, false)) |> Stream.run() {:ok, []} @@ -67,25 +71,30 @@ defmodule Archethic.Account.MemTablesLoader do @doc """ Load the transaction into the memory tables """ - @spec load_transaction(Transaction.t()) :: :ok - def load_transaction(%Transaction{ - address: address, - type: tx_type, - previous_public_key: previous_public_key, - validation_stamp: %ValidationStamp{ - timestamp: timestamp, - protocol_version: protocol_version, - ledger_operations: %LedgerOperations{ - fee: fee, - unspent_outputs: unspent_outputs, - transaction_movements: transaction_movements + @spec load_transaction(Transaction.t(), boolean()) :: :ok + def load_transaction( + %Transaction{ + address: address, + type: tx_type, + previous_public_key: previous_public_key, + validation_stamp: %ValidationStamp{ + timestamp: timestamp, + protocol_version: protocol_version, + ledger_operations: %LedgerOperations{ + fee: fee, + unspent_outputs: unspent_outputs, + transaction_movements: transaction_movements + } } - } - }) do - previous_address = Crypto.derive_address(previous_public_key) - - UCOLedger.spend_all_unspent_outputs(previous_address) - TokenLedger.spend_all_unspent_outputs(previous_address) + }, + io_transaction? + ) do + unless io_transaction? do + previous_address = Crypto.derive_address(previous_public_key) + + UCOLedger.spend_all_unspent_outputs(previous_address) + TokenLedger.spend_all_unspent_outputs(previous_address) + end burn_storage_nodes = Election.storage_nodes( diff --git a/lib/archethic/contracts/loader.ex b/lib/archethic/contracts/loader.ex index b4686a3751..29247d0956 100644 --- a/lib/archethic/contracts/loader.ex +++ b/lib/archethic/contracts/loader.ex @@ -27,17 +27,19 @@ defmodule Archethic.Contracts.Loader do def init(_opts) do DB.list_last_transaction_addresses() |> Stream.map(fn address -> - {:ok, tx} = - DB.get_transaction(address, [ - :address, - :previous_public_key, - :data, - validation_stamp: [:timestamp] - ]) - - tx + DB.get_transaction(address, [ + :address, + :previous_public_key, + :data, + validation_stamp: [:timestamp] + ]) end) - |> Stream.filter(&(&1.data.code != "")) + |> Stream.filter(fn + {:ok, %Transaction{data: %TransactionData{code: ""}}} -> false + {:error, _} -> false + _ -> true + end) + |> Stream.map(fn {:ok, tx} -> tx end) |> Stream.each(&load_transaction(&1, true)) |> Stream.run() diff --git a/lib/archethic/db.ex b/lib/archethic/db.ex index b638bd9aac..ee5efd4e14 100644 --- a/lib/archethic/db.ex +++ b/lib/archethic/db.ex @@ -13,6 +13,8 @@ defmodule Archethic.DB do use Knigge, otp_app: :archethic, default: EmbeddedImpl + @type storage_type() :: :chain | :io + @callback get_transaction(address :: binary(), fields :: list()) :: {:ok, Transaction.t()} | {:error, :transaction_not_exists} @callback get_beacon_summary(summary_address :: binary()) :: @@ -24,12 +26,13 @@ defmodule Archethic.DB do fields :: list(), opts :: [paging_state: nil | binary(), after: DateTime.t()] ) :: Enumerable.t() - @callback write_transaction(Transaction.t()) :: :ok + @callback write_transaction(Transaction.t(), storage_type()) :: :ok @callback write_beacon_summary(Summary.t()) :: :ok @callback clear_beacon_summaries() :: :ok @callback write_beacon_summaries_aggregate(SummaryAggregate.t()) :: :ok @callback write_transaction_chain(Enumerable.t()) :: :ok @callback list_transactions(fields :: list()) :: Enumerable.t() + @callback list_io_transactions(fields :: list()) :: Enumerable.t() @callback add_last_transaction_address(binary(), binary(), DateTime.t()) :: :ok @callback list_last_transaction_addresses() :: Enumerable.t() @@ -61,7 +64,7 @@ defmodule Archethic.DB do @callback get_latest_burned_fees() :: non_neg_integer() @callback get_nb_transactions() :: non_neg_integer() - @callback transaction_exists?(binary()) :: boolean() + @callback transaction_exists?(binary(), storage_type()) :: boolean() @callback register_p2p_summary(list({Crypto.key(), boolean(), float(), DateTime.t()})) :: :ok diff --git a/lib/archethic/db/embedded_impl.ex b/lib/archethic/db/embedded_impl.ex index 0136e78383..91800661c0 100644 --- a/lib/archethic/db/embedded_impl.ex +++ b/lib/archethic/db/embedded_impl.ex @@ -9,6 +9,8 @@ defmodule Archethic.DB.EmbeddedImpl do alias Archethic.Crypto + alias Archethic.DB + alias __MODULE__.BootstrapInfo alias __MODULE__.ChainIndex alias __MODULE__.ChainReader @@ -74,6 +76,9 @@ defmodule Archethic.DB.EmbeddedImpl do Enum.each(sorted_chain, fn tx -> unless ChainIndex.transaction_exists?(tx.address, db_path()) do ChainWriter.append_transaction(genesis_address, tx) + + # Delete IO transaction if it exists as it is now stored as a chain + delete_io_transaction(tx.address) end end) end @@ -81,31 +86,44 @@ defmodule Archethic.DB.EmbeddedImpl do @doc """ Write a single transaction and append it to its chain """ - @spec write_transaction(Transaction.t()) :: :ok - def write_transaction(tx = %Transaction{}) do + @spec write_transaction(Transaction.t(), DB.storage_type()) :: :ok + def write_transaction(tx, storage_type \\ :chain) + + def write_transaction(tx = %Transaction{}, :chain) do if ChainIndex.transaction_exists?(tx.address, db_path()) do {:error, :transaction_already_exists} else previous_address = Transaction.previous_address(tx) - case ChainIndex.get_tx_entry(previous_address, db_path()) do - {:ok, %{genesis_address: genesis_address}} -> - do_write_transaction(genesis_address, tx) + genesis_address = + case ChainIndex.get_tx_entry(previous_address, db_path()) do + {:ok, %{genesis_address: genesis_address}} -> + genesis_address - {:error, :not_exists} -> - ChainWriter.append_transaction(previous_address, tx) - end + {:error, :not_exists} -> + previous_address + end + + ChainWriter.append_transaction(genesis_address, tx) + + # Delete IO transaction if it exists as it is now stored as a chain + delete_io_transaction(tx.address) end end - defp do_write_transaction(genesis_address, tx) do - if ChainIndex.transaction_exists?(tx.address, db_path()) do + def write_transaction(tx = %Transaction{}, :io) do + if ChainIndex.transaction_exists?(tx.address, :io, db_path()) do {:error, :transaction_already_exists} else - ChainWriter.append_transaction(genesis_address, tx) + ChainWriter.write_io_transaction(tx, db_path()) end end + defp delete_io_transaction(address) do + ChainWriter.io_path(db_path(), address) |> File.rm() + :ok + end + @doc """ Write a beacon summary in DB """ @@ -137,9 +155,9 @@ defmodule Archethic.DB.EmbeddedImpl do @doc """ Determine if the transaction exists or not """ - @spec transaction_exists?(address :: binary()) :: boolean() - def transaction_exists?(address) when is_binary(address) do - ChainIndex.transaction_exists?(address, db_path()) + @spec transaction_exists?(address :: binary(), storage_type :: DB.storage_type()) :: boolean() + def transaction_exists?(address, storage_type) when is_binary(address) do + ChainIndex.transaction_exists?(address, storage_type, db_path()) end @doc """ @@ -278,7 +296,7 @@ defmodule Archethic.DB.EmbeddedImpl do end @doc """ - List all the transactions + List all the transactions in chain storage """ @spec list_transactions(fields :: list()) :: Enumerable.t() | list(Transaction.t()) def list_transactions(fields \\ []) when is_list(fields) do @@ -286,6 +304,14 @@ defmodule Archethic.DB.EmbeddedImpl do |> Stream.flat_map(&ChainReader.stream_scan_chain(&1, nil, fields, db_path())) end + @doc """ + List all the transactions in io storage + """ + @spec list_io_transactions(fields :: list()) :: Enumerable.t() | list(Transaction.t()) + def list_io_transactions(fields \\ []) do + ChainReader.list_io_transactions(fields, db_path()) + end + @doc """ List all the last transaction chain addresses """ diff --git a/lib/archethic/db/embedded_impl/chain_index.ex b/lib/archethic/db/embedded_impl/chain_index.ex index 867c706f30..2b07514292 100644 --- a/lib/archethic/db/embedded_impl/chain_index.ex +++ b/lib/archethic/db/embedded_impl/chain_index.ex @@ -7,6 +7,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do @vsn Mix.Project.config()[:version] alias Archethic.Crypto + alias Archethic.DB alias Archethic.DB.EmbeddedImpl.ChainWriter alias Archethic.TransactionChain.Transaction @@ -196,14 +197,18 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do @doc """ Determine if a transaction exists """ - @spec transaction_exists?(binary(), String.t()) :: boolean() - def transaction_exists?(address = <<_::8, _::8, _subset::8, _digest::binary>>, db_path) do + @spec transaction_exists?(binary(), DB.storage_type(), String.t()) :: boolean() + def transaction_exists?(address, storage_type \\ :chain, db_path) + + def transaction_exists?(address, storage_type, db_path) do case get_tx_entry(address, db_path) do {:ok, _} -> true {:error, :not_exists} -> - false + if storage_type == :io, + do: ChainWriter.io_path(db_path, address) |> File.exists?(), + else: false end end diff --git a/lib/archethic/db/embedded_impl/chain_reader.ex b/lib/archethic/db/embedded_impl/chain_reader.ex index 57cca625f6..ebffc3e554 100644 --- a/lib/archethic/db/embedded_impl/chain_reader.ex +++ b/lib/archethic/db/embedded_impl/chain_reader.ex @@ -139,6 +139,60 @@ defmodule Archethic.DB.EmbeddedImpl.ChainReader do end end + @doc """ + List all the transactions in io storage + """ + @spec list_io_transactions(fields :: list(), db_path :: String.t()) :: + Enumerable.t() | list(Transaction.t()) + def list_io_transactions(fields, db_path) do + io_transactions_path = + ChainWriter.base_io_path(db_path) + |> Path.join("*") + |> Path.wildcard() + + Stream.resource( + fn -> io_transactions_path end, + fn + [filepath | rest] -> {[read_io_transaction(filepath, fields)], rest} + [] -> {:halt, nil} + end, + fn _ -> :ok end + ) + end + + defp read_io_transaction(filepath, fields) do + # Open the file as the position from the transaction in the chain file + fd = File.open!(filepath, [:binary, :read]) + + {:ok, <>} = :file.read(fd, 8) + column_names = fields_to_column_names(fields) + + # Ensure the validation stamp's protocol version is retrieved if we fetch validation stamp fields + has_validation_stamp_fields? = + Enum.any?(column_names, &String.starts_with?(&1, "validation_stamp.")) + + has_validation_stamp_protocol_field? = + Enum.any?(column_names, &(&1 == "validation_stamp.protocol_version")) + + column_names = + if has_validation_stamp_fields? and !has_validation_stamp_protocol_field? do + ["validation_stamp.protocol_version" | column_names] + else + column_names + end + + # Read the transaction and extract requested columns from the fields arg + tx = + fd + |> read_transaction(column_names, size, 0) + |> Enum.into(%{}) + |> decode_transaction_columns(version) + + :file.close(fd) + + tx + end + defp process_get_chain(fd, address, fields, opts, db_path) do # Set the file cursor position to the paging state case Keyword.get(opts, :paging_state) do diff --git a/lib/archethic/db/embedded_impl/chain_writer.ex b/lib/archethic/db/embedded_impl/chain_writer.ex index f746548881..8ee44d522a 100644 --- a/lib/archethic/db/embedded_impl/chain_writer.ex +++ b/lib/archethic/db/embedded_impl/chain_writer.ex @@ -31,6 +31,28 @@ defmodule Archethic.DB.EmbeddedImpl.ChainWriter do GenServer.call(pid, {:append_tx, genesis_address, tx}) end + @doc """ + write an io transaction in a file name by it's address + """ + @spec write_io_transaction(Transaction.t(), String.t()) :: :ok + def write_io_transaction(tx = %Transaction{address: address}, db_path) do + start = System.monotonic_time() + + filename = io_path(db_path, address) + + data = Encoding.encode(tx) + + File.write!( + filename, + data, + [:exclusive, :binary] + ) + + :telemetry.execute([:archethic, :db], %{duration: System.monotonic_time() - start}, %{ + query: "write_io_transaction" + }) + end + @doc """ Write a beacon summary in a new file """ @@ -99,6 +121,10 @@ defmodule Archethic.DB.EmbeddedImpl.ChainWriter do |> base_chain_path() |> File.mkdir_p!() + path + |> base_io_path() + |> File.mkdir_p!() + path |> base_beacon_path() |> File.mkdir_p!() @@ -117,6 +143,15 @@ defmodule Archethic.DB.EmbeddedImpl.ChainWriter do {:reply, :ok, state} end + def handle_call( + {:write_io_transaction, tx}, + _from, + state = %{db_path: db_path} + ) do + write_io_transaction(tx, db_path) + {:reply, :ok, state} + end + def terminate(_reason, _state = %{partition: partition}) do :ets.delete(:archethic_db_chain_writers, partition) :ignore @@ -174,6 +209,15 @@ defmodule Archethic.DB.EmbeddedImpl.ChainWriter do Path.join([base_chain_path(db_path), Base.encode16(genesis_address)]) end + @doc """ + Return the path of the io storage location + """ + @spec io_path(String.t(), binary()) :: String.t() + def io_path(db_path, address) + when is_binary(address) and is_binary(db_path) do + Path.join([base_io_path(db_path), Base.encode16(address)]) + end + @doc """ Return the chain base path """ @@ -182,6 +226,14 @@ defmodule Archethic.DB.EmbeddedImpl.ChainWriter do Path.join([db_path, "chains"]) end + @doc """ + Return the io base path + """ + @spec base_io_path(String.t()) :: String.t() + def base_io_path(db_path) do + Path.join([db_path, "io"]) + end + @doc """ Return the path of the beacon summary storage location """ diff --git a/lib/archethic/oracle_chain/scheduler.ex b/lib/archethic/oracle_chain/scheduler.ex index 2985eb1a33..23b35794e7 100644 --- a/lib/archethic/oracle_chain/scheduler.ex +++ b/lib/archethic/oracle_chain/scheduler.ex @@ -4,7 +4,7 @@ defmodule Archethic.OracleChain.Scheduler do """ alias Archethic.Crypto - alias Archethic.DB + alias Archethic.Election alias Archethic.P2P @@ -307,7 +307,7 @@ defmodule Archethic.OracleChain.Scheduler do tx = build_oracle_transaction(summary_date, index, new_oracle_data) with {:empty, false} <- {:empty, Enum.empty?(new_oracle_data)}, - {:exists, false} <- {:exists, DB.transaction_exists?(tx.address)}, + {:exists, false} <- {:exists, TransactionChain.transaction_exists?(tx.address)}, {:trigger, true} <- {:trigger, trigger_node?(storage_nodes)} do send_polling_transaction(tx) :keep_state_and_data @@ -364,7 +364,7 @@ defmodule Archethic.OracleChain.Scheduler do storage_nodes = tx_address |> Election.storage_nodes(authorized_nodes) watcher_pid = - with {:exists, false} <- {:exists, DB.transaction_exists?(tx_address)}, + with {:exists, false} <- {:exists, TransactionChain.transaction_exists?(tx_address)}, {:trigger, true} <- {:trigger, trigger_node?(storage_nodes)} do Logger.debug("Oracle transaction summary sending", transaction_address: Base.encode16(tx_address), @@ -699,7 +699,7 @@ defmodule Archethic.OracleChain.Scheduler do next_pub ) - if DB.transaction_exists?(tx.address) do + if TransactionChain.transaction_exists?(tx.address) do Logger.debug( "Transaction Already Exists:oracle summary transaction - aggregation: #{inspect(aggregated_content)}", transaction_address: Base.encode16(tx.address), diff --git a/lib/archethic/replication.ex b/lib/archethic/replication.ex index 98f397ea89..ce35cbc711 100644 --- a/lib/archethic/replication.ex +++ b/lib/archethic/replication.ex @@ -111,7 +111,7 @@ defmodule Archethic.Replication do TransactionChain.write_transaction(tx) - :ok = ingest_transaction(tx) + :ok = ingest_transaction(tx, false) Logger.info("Replication finished", transaction_address: Base.encode16(address), @@ -158,7 +158,7 @@ defmodule Archethic.Replication do }, self_repair? \\ false ) do - if TransactionChain.transaction_exists?(address) do + if TransactionChain.transaction_exists?(address, :io) do Logger.warning("Transaction already exists", transaction_address: Base.encode16(address), transaction_type: type @@ -175,8 +175,8 @@ defmodule Archethic.Replication do case TransactionValidator.validate(tx, self_repair?) do :ok -> - :ok = TransactionChain.write_transaction(tx) - ingest_transaction(tx) + :ok = TransactionChain.write_transaction(tx, :io) + ingest_transaction(tx, true) Logger.info("Replication finished", transaction_address: Base.encode16(address), @@ -525,13 +525,13 @@ defmodule Archethic.Replication do - Transactions with smart contract deploy instances of them or can put in pending state waiting approval signatures - Code approval transactions may trigger the TestNets deployments or hot-reloads """ - @spec ingest_transaction(Transaction.t()) :: :ok - def ingest_transaction(tx = %Transaction{}) do + @spec ingest_transaction(Transaction.t(), boolean()) :: :ok + def ingest_transaction(tx = %Transaction{}, io_transaction?) do TransactionChain.load_transaction(tx) Crypto.load_transaction(tx) P2P.load_transaction(tx) SharedSecrets.load_transaction(tx) - Account.load_transaction(tx) + Account.load_transaction(tx, io_transaction?) Contracts.load_transaction(tx) OracleChain.load_transaction(tx) Reward.load_transaction(tx) diff --git a/lib/archethic/self_repair/sync.ex b/lib/archethic/self_repair/sync.ex index bdf8738a7f..5ddca9a183 100644 --- a/lib/archethic/self_repair/sync.ex +++ b/lib/archethic/self_repair/sync.ex @@ -232,7 +232,7 @@ defmodule Archethic.SelfRepair.Sync do transactions_to_sync = transaction_summaries - |> Enum.reject(&TransactionChain.transaction_exists?(&1.address)) + |> Enum.reject(&TransactionChain.transaction_exists?(&1.address, :io)) |> Enum.filter(&TransactionHandler.download_transaction?/1) synchronize_transactions(transactions_to_sync, download_nodes) diff --git a/lib/archethic/transaction_chain.ex b/lib/archethic/transaction_chain.ex index ee6ec16b6c..3bbd9875f9 100644 --- a/lib/archethic/transaction_chain.ex +++ b/lib/archethic/transaction_chain.ex @@ -58,6 +58,12 @@ defmodule Archethic.TransactionChain do @spec list_all(fields :: list()) :: Enumerable.t() defdelegate list_all(fields \\ []), to: DB, as: :list_transactions + @doc """ + List all the io transactions stored + """ + @spec list_io_transactions(fields :: list()) :: Enumerable.t() + defdelegate list_io_transactions(fields \\ []), to: DB + @doc """ List all the transaction for a given transaction type sorted by timestamp in descent order """ @@ -172,14 +178,15 @@ defmodule Archethic.TransactionChain do @doc """ Persist only one transaction """ - @spec write_transaction(Transaction.t()) :: :ok + @spec write_transaction(Transaction.t(), DB.storage_type()) :: :ok def write_transaction( tx = %Transaction{ address: address, type: type - } + }, + storage_type \\ :chain ) do - DB.write_transaction(tx) + DB.write_transaction(tx, storage_type) KOLedger.remove_transaction(address) Logger.info("Transaction stored", @@ -247,8 +254,8 @@ defmodule Archethic.TransactionChain do @doc """ Determine if the transaction exists """ - @spec transaction_exists?(binary()) :: boolean() - defdelegate transaction_exists?(address), to: DB + @spec transaction_exists?(binary(), DB.storage_type()) :: boolean() + defdelegate transaction_exists?(address, storage_type \\ :chain), to: DB @doc """ Return the size of transaction chain diff --git a/lib/archethic/utils/detect_node_responsiveness.ex b/lib/archethic/utils/detect_node_responsiveness.ex index a4f8d0c236..9810bc99b5 100644 --- a/lib/archethic/utils/detect_node_responsiveness.ex +++ b/lib/archethic/utils/detect_node_responsiveness.ex @@ -4,9 +4,11 @@ defmodule Archethic.Utils.DetectNodeResponsiveness do """ @default_timeout Application.compile_env(:archethic, __MODULE__, []) |> Keyword.get(:timeout, 5_000) - alias Archethic.DB + alias Archethic.Mining + alias Archethic.TransactionChain + use GenServer @vsn Mix.Project.config()[:version] require Logger @@ -43,7 +45,7 @@ defmodule Archethic.Utils.DetectNodeResponsiveness do timeout: timeout } ) do - with {:exists, false} <- {:exists, DB.transaction_exists?(address)}, + with {:exists, false} <- {:exists, TransactionChain.transaction_exists?(address)}, {:mining, false} <- {:mining, Mining.processing?(address)}, {:remaining, true} <- {:remaining, count < max_retry} do Logger.info("calling replay fn with count=#{count}", diff --git a/test/archethic/account/mem_tables_loader_test.exs b/test/archethic/account/mem_tables_loader_test.exs index 2da3cffd6d..27291f2fba 100644 --- a/test/archethic/account/mem_tables_loader_test.exs +++ b/test/archethic/account/mem_tables_loader_test.exs @@ -107,7 +107,9 @@ defmodule Archethic.Account.MemTablesLoaderTest do }) timestamp = DateTime.utc_now() |> DateTime.truncate(:millisecond) - assert :ok = MemTablesLoader.load_transaction(create_transaction(timestamp)) + + assert :ok = + MemTablesLoader.load_transaction(create_transaction(timestamp, "@Charlie3"), false) assert [ %VersionedUnspentOutput{ @@ -166,8 +168,11 @@ defmodule Archethic.Account.MemTablesLoaderTest do timestamp = DateTime.utc_now() |> DateTime.truncate(:millisecond) MockDB + |> stub(:list_io_transactions, fn _fields -> + [create_transaction(timestamp, "@Charlie4")] + end) |> stub(:list_transactions, fn _fields -> - [create_transaction(timestamp)] + [create_transaction(timestamp, "@Charlie3")] end) %{timestamp: timestamp} @@ -203,6 +208,14 @@ defmodule Archethic.Account.MemTablesLoaderTest do type: :UCO, timestamp: ^timestamp } + }, + %VersionedUnspentOutput{ + unspent_output: %UnspentOutput{ + from: "@Charlie4", + amount: 3_400_000_000, + type: :UCO, + timestamp: ^timestamp + } } ] = UCOLedger.get_unspent_outputs("@Tom4") @@ -214,14 +227,22 @@ defmodule Archethic.Account.MemTablesLoaderTest do type: {:token, "@CharlieToken", 0}, timestamp: ^timestamp } + }, + %VersionedUnspentOutput{ + unspent_output: %UnspentOutput{ + from: "@Charlie4", + amount: 1_000_000_000, + type: {:token, "@CharlieToken", 0}, + timestamp: ^timestamp + } } ] = TokenLedger.get_unspent_outputs("@Bob3") end end - defp create_transaction(timestamp) do + defp create_transaction(timestamp, address) do %Transaction{ - address: "@Charlie3", + address: address, previous_public_key: "Charlie2", validation_stamp: %ValidationStamp{ protocol_version: ArchethicCase.current_protocol_version(), @@ -263,9 +284,8 @@ defmodule Archethic.Account.MemTablesLoaderTest do DateTime.utc_now() |> DateTime.add(-86_400) |> DateTime.truncate(:millisecond) assert :ok = - MemTablesLoader.load_transaction( - create_reward_transaction(timestamp, validation_time) - ) + create_reward_transaction(timestamp, validation_time) + |> MemTablesLoader.load_transaction(false) # uco ledger assert [ diff --git a/test/archethic/bootstrap/network_init_test.exs b/test/archethic/bootstrap/network_init_test.exs index fab68996ea..45bd774a9a 100644 --- a/test/archethic/bootstrap/network_init_test.exs +++ b/test/archethic/bootstrap/network_init_test.exs @@ -228,7 +228,7 @@ defmodule Archethic.Bootstrap.NetworkInitTest do me = self() MockDB - |> stub(:write_transaction, fn ^tx -> + |> stub(:write_transaction, fn ^tx, _ -> send(me, :write_transaction) :ok end) @@ -277,7 +277,7 @@ defmodule Archethic.Bootstrap.NetworkInitTest do me = self() MockDB - |> expect(:write_transaction, fn tx -> + |> expect(:write_transaction, fn tx, _ -> send(me, {:transaction, tx}) :ok end) @@ -429,7 +429,7 @@ defmodule Archethic.Bootstrap.NetworkInitTest do me = self() MockDB - |> expect(:write_transaction, fn tx -> + |> expect(:write_transaction, fn tx, _ -> send(me, {:transaction, tx}) :ok end) diff --git a/test/archethic/bootstrap_test.exs b/test/archethic/bootstrap_test.exs index 0a3c1ac584..5c6f6067a0 100644 --- a/test/archethic/bootstrap_test.exs +++ b/test/archethic/bootstrap_test.exs @@ -263,7 +263,7 @@ defmodule Archethic.BootstrapTest do validated_tx = %{tx | validation_stamp: stamp} :ok = TransactionChain.write([validated_tx]) - :ok = Replication.ingest_transaction(validated_tx) + :ok = Replication.ingest_transaction(validated_tx, false) {:ok, %Ok{}} @@ -549,9 +549,9 @@ defmodule Archethic.BootstrapTest do ^addr0 -> {addr2, DateTime.utc_now()} end) |> stub(:transaction_exists?, fn - ^addr4 -> false - ^addr3 -> false - ^addr2 -> true + ^addr4, _ -> false + ^addr3, _ -> false + ^addr2, _ -> true end) |> expect(:get_transaction, fn ^addr3, _ -> {:error, :transaction_not_exists} @@ -562,7 +562,7 @@ defmodule Archethic.BootstrapTest do send(me, {:write_transaction_chain, txn_list}) :ok end) - |> expect(:write_transaction, fn _tx -> + |> expect(:write_transaction, fn _tx, _ -> :ok end) diff --git a/test/archethic/db/embedded_impl_test.exs b/test/archethic/db/embedded_impl_test.exs index 271f8deba2..10e621ceeb 100644 --- a/test/archethic/db/embedded_impl_test.exs +++ b/test/archethic/db/embedded_impl_test.exs @@ -122,18 +122,71 @@ defmodule Archethic.DB.EmbeddedTest do genesis_address: ^genesis_address }} = ChainIndex.get_tx_entry(tx2.address, db_path) end + + test "should write transaction in io storage", %{db_path: db_path} do + tx1 = TransactionFactory.create_valid_transaction() + :ok = EmbeddedImpl.write_transaction(tx1, :io) + + filename = ChainWriter.io_path(db_path, tx1.address) + + assert File.exists?(filename) + + contents = File.read!(filename) + + assert contents == Encoding.encode(tx1) + end + + test "should delete transaction in io storage after writing it in chain storage", %{ + db_path: db_path + } do + tx1 = TransactionFactory.create_valid_transaction() + :ok = EmbeddedImpl.write_transaction(tx1, :io) + + filename_io = ChainWriter.io_path(db_path, tx1.address) + + assert File.exists?(filename_io) + + :ok = EmbeddedImpl.write_transaction(tx1) + + genesis_address = Transaction.previous_address(tx1) + filename_chain = ChainWriter.chain_path(db_path, genesis_address) + + assert File.exists?(filename_chain) + assert !File.exists?(filename_io) + end end - describe "transaction_exists?/1" do - test "should return true when the transaction is present" do + describe "transaction_exists?/2" do + test "should return true when the transaction is present in chain storage" do tx1 = TransactionFactory.create_valid_transaction() :ok = EmbeddedImpl.write_transaction_chain([tx1]) - assert EmbeddedImpl.transaction_exists?(tx1.address) + assert EmbeddedImpl.transaction_exists?(tx1.address, :chain) end test "should return false when the transaction is present" do - assert !EmbeddedImpl.transaction_exists?(:crypto.strong_rand_bytes(32)) + assert !EmbeddedImpl.transaction_exists?(:crypto.strong_rand_bytes(32), :chain) + end + + test "should return false when the transaction is not present in chain storage but in io storage" do + tx1 = TransactionFactory.create_valid_transaction() + :ok = EmbeddedImpl.write_transaction(tx1, :io) + + assert !EmbeddedImpl.transaction_exists?(tx1.address, :chain) + end + + test "should return true when the transaction is present in io storage" do + tx1 = TransactionFactory.create_valid_transaction() + :ok = EmbeddedImpl.write_transaction(tx1, :io) + + assert EmbeddedImpl.transaction_exists?(tx1.address, :io) + end + + test "should return true when the transaction is present in chain storage and asking for io storage" do + tx1 = TransactionFactory.create_valid_transaction() + :ok = EmbeddedImpl.write_transaction(tx1) + + assert EmbeddedImpl.transaction_exists?(tx1.address, :io) end end diff --git a/test/archethic/replication_test.exs b/test/archethic/replication_test.exs index fa95916df8..6695310b0b 100644 --- a/test/archethic/replication_test.exs +++ b/test/archethic/replication_test.exs @@ -83,7 +83,7 @@ defmodule Archethic.ReplicationTest do # send(me, :replicated) # :ok # end) - |> expect(:write_transaction, fn ^tx -> + |> expect(:write_transaction, fn ^tx, _ -> send(me, :replicated) :ok end) @@ -145,7 +145,7 @@ defmodule Archethic.ReplicationTest do tx = create_valid_transaction(unspent_outputs) MockDB - |> expect(:write_transaction, fn _ -> + |> expect(:write_transaction, fn _, _ -> send(me, :replicated) :ok end) diff --git a/test/archethic/reward_test.exs b/test/archethic/reward_test.exs index 255977e1dd..a542a86f2a 100644 --- a/test/archethic/reward_test.exs +++ b/test/archethic/reward_test.exs @@ -138,7 +138,7 @@ defmodule Archethic.RewardTest do end test "Balance Should be updated with UCO ,for reward movements " do - Enum.each(get_reward_transactions(), &AccountTablesLoader.load_transaction(&1)) + Enum.each(get_reward_transactions(), &AccountTablesLoader.load_transaction(&1, false)) # @Ada1 # from alen2: 2uco, dan2: 19uco, rewardtoken1: 50, rewardtoken2: 50 @@ -365,7 +365,7 @@ defmodule Archethic.RewardTest do end test "Node Rewards Should not be minted" do - AccountTablesLoader.load_transaction(get_node_reward_txns()) + AccountTablesLoader.load_transaction(get_node_reward_txns(), false) assert %{ uco: 0, diff --git a/test/archethic/self_repair/repair_worker_test.exs b/test/archethic/self_repair/repair_worker_test.exs index 487520ae2d..2d5bd2f3b2 100644 --- a/test/archethic/self_repair/repair_worker_test.exs +++ b/test/archethic/self_repair/repair_worker_test.exs @@ -57,11 +57,11 @@ defmodule Archethic.SelfRepair.RepairWorkerTest do MockDB |> stub(:transaction_exists?, fn - "Bob2" -> + "Bob2", _ -> send(me, :exists_bob3) true - _ -> + _, _ -> false end) @@ -88,7 +88,7 @@ defmodule Archethic.SelfRepair.RepairWorkerTest do test "add_message/1 should add new addresses in GenServer state" do MockDB - |> stub(:transaction_exists?, fn _ -> Process.sleep(100) end) + |> stub(:transaction_exists?, fn _, _ -> Process.sleep(100) end) {:ok, pid} = RepairWorker.start_link( diff --git a/test/archethic/self_repair/sync/transaction_handler_test.exs b/test/archethic/self_repair/sync/transaction_handler_test.exs index 23eb71b6f9..66848026a7 100644 --- a/test/archethic/self_repair/sync/transaction_handler_test.exs +++ b/test/archethic/self_repair/sync/transaction_handler_test.exs @@ -173,7 +173,7 @@ defmodule Archethic.SelfRepair.Sync.TransactionHandlerTest do end) MockDB - |> stub(:write_transaction, fn ^tx -> + |> stub(:write_transaction, fn ^tx, _ -> send(me, :transaction_replicated) :ok end) diff --git a/test/archethic/self_repair/sync_test.exs b/test/archethic/self_repair/sync_test.exs index fe2b0b43fe..cc0a40ffbe 100644 --- a/test/archethic/self_repair/sync_test.exs +++ b/test/archethic/self_repair/sync_test.exs @@ -168,7 +168,7 @@ defmodule Archethic.SelfRepair.SyncTest do me = self() MockDB - |> stub(:write_transaction, fn ^tx -> + |> stub(:write_transaction, fn ^tx, _ -> send(me, :storage) :ok end) @@ -301,7 +301,7 @@ defmodule Archethic.SelfRepair.SyncTest do me = self() MockDB - |> stub(:write_transaction, fn ^transfer_tx -> + |> stub(:write_transaction, fn ^transfer_tx, _ -> send(me, :transaction_stored) :ok end) diff --git a/test/archethic/utils/detect_node_responsiveness_test.exs b/test/archethic/utils/detect_node_responsiveness_test.exs index 6de7122d86..a843c6dac3 100644 --- a/test/archethic/utils/detect_node_responsiveness_test.exs +++ b/test/archethic/utils/detect_node_responsiveness_test.exs @@ -110,7 +110,7 @@ defmodule Archethic.Utils.DetectNodeResponsivenessTest do Process.monitor(pid) MockDB - |> stub(:transaction_exists?, fn ^address -> + |> stub(:transaction_exists?, fn ^address, _ -> false end) @@ -170,7 +170,7 @@ defmodule Archethic.Utils.DetectNodeResponsivenessTest do {:ok, pid} = DetectNodeResponsiveness.start_link(address, 2, replaying_fn, @timeout) MockDB - |> stub(:transaction_exists?, fn ^address -> + |> stub(:transaction_exists?, fn ^address, _ -> Process.send_after(me, :transaction_stored, 50) true end) @@ -221,10 +221,10 @@ defmodule Archethic.Utils.DetectNodeResponsivenessTest do {:ok, pid} = DetectNodeResponsiveness.start_link(address, 2, replaying_fn, @timeout) MockDB - |> expect(:transaction_exists?, fn ^address -> + |> expect(:transaction_exists?, fn ^address, _ -> false end) - |> expect(:transaction_exists?, fn ^address -> + |> expect(:transaction_exists?, fn ^address, _ -> Process.send_after(me, :transaction_stored, 50) true end) @@ -276,7 +276,7 @@ defmodule Archethic.Utils.DetectNodeResponsivenessTest do {:ok, pid} = DetectNodeResponsiveness.start_link(address, 2, replaying_fn, @timeout) MockDB - |> stub(:transaction_exists?, fn ^address -> + |> stub(:transaction_exists?, fn ^address, _ -> false end) @@ -327,7 +327,7 @@ defmodule Archethic.Utils.DetectNodeResponsivenessTest do {:ok, pid} = DetectNodeResponsiveness.start_link(address, 2, replaying_fn, @timeout) MockDB - |> stub(:transaction_exists?, fn ^address -> + |> stub(:transaction_exists?, fn ^address, _ -> false end) diff --git a/test/support/template.ex b/test/support/template.ex index c5713eeb68..98e7fa6f16 100644 --- a/test/support/template.ex +++ b/test/support/template.ex @@ -40,7 +40,7 @@ defmodule ArchethicCase do MockDB |> stub(:list_transactions, fn _ -> [] end) - |> stub(:write_transaction, fn _ -> :ok end) + |> stub(:write_transaction, fn _, _ -> :ok end) |> stub(:write_transaction_chain, fn _ -> :ok end) |> stub(:get_transaction, fn _, _ -> {:error, :transaction_not_exists} end) |> stub(:get_transaction_chain, fn _, _, _ -> {[], false, nil} end) @@ -56,7 +56,7 @@ defmodule ArchethicCase do |> stub(:count_transactions_by_type, fn _ -> 0 end) |> stub(:list_addresses_by_type, fn _ -> [] end) |> stub(:list_transactions, fn _ -> [] end) - |> stub(:transaction_exists?, fn _ -> false end) + |> stub(:transaction_exists?, fn _, _ -> false end) |> stub(:register_p2p_summary, fn _ -> :ok end) |> stub(:get_last_p2p_summaries, fn -> [] end) |> stub(:get_latest_tps, fn -> 0.0 end) From 5fa619bf71484c6bde13d44a5763f873929b0bf3 Mon Sep 17 00:00:00 2001 From: Neylix Date: Fri, 2 Dec 2022 17:00:39 +0100 Subject: [PATCH 8/9] Fix mix format error --- lib/archethic/p2p/client/connection.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/archethic/p2p/client/connection.ex b/lib/archethic/p2p/client/connection.ex index 1997930b10..016ecc3b17 100644 --- a/lib/archethic/p2p/client/connection.ex +++ b/lib/archethic/p2p/client/connection.ex @@ -213,8 +213,9 @@ defmodule Archethic.P2P.Client.Connection do # try to connect asynchronously so it does not block the messages coming in # Task.async/1 will send a {:info, {ref, result}} message to the connection process me = self() + Task.async(fn -> - case transport.handle_connect(ip, port) do + case transport.handle_connect(ip, port) do {:ok, socket} when is_port(socket) -> :gen_tcp.controlling_process(socket, me) {:ok, socket} @@ -222,6 +223,7 @@ defmodule Archethic.P2P.Client.Connection do # only used for tests (make_ref()) {:ok, socket} -> {:ok, socket} + {:error, _} = e -> e end From 0d5fbd630670b0bf1bef7503afeedf86d94b2081 Mon Sep 17 00:00:00 2001 From: Samuel Manzanera Date: Fri, 2 Dec 2022 17:25:53 +0100 Subject: [PATCH 9/9] Improve smart contract interpreter(#628) To simplify the interpreter, it is now built of two interpreters: * Add Condition's interpreter * Add Action's interpreter Hence, the validation is easier to handle and to customize. This PR brings other improvements: * Update interpreters * Uniform the binary/hex handling in smart contract * Fix SC call inputs serialization * Clean inputs ETS table for SC calls * Clear pending transactions table Co-authored-by: Neylix --- .dialyzer_ignore.exs | 2 - .github/workflows/ci.yml | 2 +- lib/archethic/account/mem_tables_loader.ex | 4 + lib/archethic/contracts.ex | 44 +- lib/archethic/contracts/contract.ex | 54 +- .../contracts/contract/conditions.ex | 5 +- lib/archethic/contracts/contract/constants.ex | 67 +- lib/archethic/contracts/contract/trigger.ex | 19 - lib/archethic/contracts/interpreter.ex | 1387 +---------------- lib/archethic/contracts/interpreter/action.ex | 338 ++++ .../contracts/interpreter/condition.ex | 601 +++++++ .../contracts/interpreter/library.ex | 35 +- .../interpreter/transaction_statements.ex | 16 +- lib/archethic/contracts/interpreter/utils.ex | 547 +++++++ lib/archethic/contracts/loader.ex | 63 +- lib/archethic/contracts/transaction_lookup.ex | 72 +- lib/archethic/contracts/worker.ex | 132 +- lib/archethic/db.ex | 6 +- lib/archethic/db/embedded_impl.ex | 7 +- .../db/embedded_impl/inputs_reader.ex | 2 +- .../db/embedded_impl/inputs_writer.ex | 9 +- .../mining/pending_transaction_validation.ex | 2 +- lib/archethic/mining/proof_of_work.ex | 13 +- lib/archethic/p2p/message.ex | 7 +- lib/archethic/transaction_chain.ex | 6 + .../transaction_chain/mem_tables_loader.ex | 4 +- .../transaction/validation_stamp.ex | 24 +- .../transaction_chain/transaction_input.ex | 2 +- .../contracts/interpreter/action_test.exs | 346 ++++ .../contracts/interpreter/condition_test.exs | 280 ++++ test/archethic/contracts/interpreter_test.exs | 668 +------- test/archethic/contracts/loader_test.exs | 9 +- test/archethic/contracts/worker_test.exs | 21 +- test/archethic/contracts_test.exs | 19 +- 34 files changed, 2596 insertions(+), 2217 deletions(-) delete mode 100644 .dialyzer_ignore.exs delete mode 100644 lib/archethic/contracts/contract/trigger.ex create mode 100644 lib/archethic/contracts/interpreter/action.ex create mode 100644 lib/archethic/contracts/interpreter/condition.ex create mode 100644 lib/archethic/contracts/interpreter/utils.ex create mode 100644 test/archethic/contracts/interpreter/action_test.exs create mode 100644 test/archethic/contracts/interpreter/condition_test.exs diff --git a/.dialyzer_ignore.exs b/.dialyzer_ignore.exs deleted file mode 100644 index 0d4f101c7a..0000000000 --- a/.dialyzer_ignore.exs +++ /dev/null @@ -1,2 +0,0 @@ -[ -] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9fc8a06900..8aefa25ae8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,4 +70,4 @@ jobs: mkdir -p priv/plts mix dialyzer --plt - name: Run dialyzer - run: mix dialyzer --no-check + run: mix dialyzer --no-check --ignore-exit-status diff --git a/lib/archethic/account/mem_tables_loader.ex b/lib/archethic/account/mem_tables_loader.ex index 282da484d1..f86528b036 100644 --- a/lib/archethic/account/mem_tables_loader.ex +++ b/lib/archethic/account/mem_tables_loader.ex @@ -162,6 +162,10 @@ defmodule Archethic.Account.MemTablesLoader do } } -> TokenLedger.add_unspent_output(address, unspent_output) + + _ -> + # Ignore smart contract calls + :ignore end) end diff --git a/lib/archethic/contracts.ex b/lib/archethic/contracts.ex index b9f91cac26..402618e657 100644 --- a/lib/archethic/contracts.ex +++ b/lib/archethic/contracts.ex @@ -5,9 +5,9 @@ defmodule Archethic.Contracts do """ alias __MODULE__.Contract - alias __MODULE__.Contract.Conditions - alias __MODULE__.Contract.Constants - alias __MODULE__.Contract.Trigger + alias __MODULE__.ContractConditions, as: Conditions + alias __MODULE__.ContractConstants, as: Constants + alias __MODULE__.ConditionInterpreter alias __MODULE__.Interpreter alias __MODULE__.Loader alias __MODULE__.TransactionLookup @@ -59,9 +59,8 @@ defmodule Archethic.Contracts do contract: nil, transaction: nil }, - triggers: [ - %Trigger{ - actions: {:__block__, [], [ + triggers: %{ + {:datetime, ~U[2020-09-25 13:18:43Z]} => {:__block__, [], [ { :=, [line: 7], @@ -79,11 +78,9 @@ defmodule Archethic.Contracts do ] } ]}, - opts: [at: ~U[2020-09-25 13:18:43Z]], - type: :datetime } - ] - }} + } + } """ @spec parse(binary()) :: {:ok, Contract.t()} | {:error, binary()} def parse(contract_code) when is_binary(contract_code) do @@ -100,11 +97,10 @@ defmodule Archethic.Contracts do }) cond do - Enum.any?(triggers, &(&1.type == :transaction)) and - Conditions.empty?(transaction_conditions) -> + Map.has_key?(triggers, :transaction) and Conditions.empty?(transaction_conditions) -> {:error, "missing transaction conditions"} - Enum.any?(triggers, &(&1.type == :oracle)) and Conditions.empty?(oracle_conditions) -> + Map.has_key?(triggers, :oracle) and Conditions.empty?(oracle_conditions) -> {:error, "missing oracle conditions"} true -> @@ -168,7 +164,7 @@ defmodule Archethic.Contracts do end defp validate_conditions(inherit_conditions, constants) do - if Interpreter.valid_conditions?(inherit_conditions, constants) do + if ConditionInterpreter.valid_conditions?(inherit_conditions, constants) do :ok else Logger.error("Inherit constraints not respected") @@ -176,10 +172,12 @@ defmodule Archethic.Contracts do end end - defp validate_triggers([], _, _), do: :ok + defp validate_triggers(triggers, _next_tx, _date) when map_size(triggers) == 0, do: :ok defp validate_triggers(triggers, next_tx, date) do - if Enum.any?(triggers, &valid_from_trigger?(&1, next_tx, date)) do + if Enum.any?(triggers, fn {trigger_type, _} -> + valid_from_trigger?(trigger_type, next_tx, date) + end) do :ok else Logger.error("Transaction not processed by a valid smart contract trigger") @@ -188,7 +186,7 @@ defmodule Archethic.Contracts do end defp valid_from_trigger?( - %Trigger{type: :datetime, opts: [at: datetime]}, + {:datetime, datetime}, %Transaction{}, validation_date = %DateTime{} ) do @@ -198,21 +196,25 @@ defmodule Archethic.Contracts do end defp valid_from_trigger?( - %Trigger{type: :interval, opts: [at: interval]}, + {:interval, interval}, %Transaction{}, validation_date = %DateTime{} ) do interval - |> CronParser.parse!(true) + |> CronParser.parse!() |> CronDateChecker.matches_date?(DateTime.to_naive(validation_date)) end - defp valid_from_trigger?(%Trigger{type: :transaction}, _, _), do: true + defp valid_from_trigger?(_, _, _), do: true @doc """ List the address of the transaction which has contacted a smart contract """ - @spec list_contract_transactions(binary()) :: list({binary(), DateTime.t()}) + @spec list_contract_transactions(contract_address :: binary()) :: + list( + {transaction_address :: binary(), transaction_timestamp :: DateTime.t(), + protocol_version :: non_neg_integer()} + ) defdelegate list_contract_transactions(address), to: TransactionLookup, as: :list_contract_transactions diff --git a/lib/archethic/contracts/contract.ex b/lib/archethic/contracts/contract.ex index c562859e7c..d566901cc8 100644 --- a/lib/archethic/contracts/contract.ex +++ b/lib/archethic/contracts/contract.ex @@ -3,9 +3,8 @@ defmodule Archethic.Contracts.Contract do Represents a smart contract """ - alias Archethic.Contracts.Contract.Conditions - alias Archethic.Contracts.Contract.Constants - alias Archethic.Contracts.Contract.Trigger + alias Archethic.Contracts.ContractConditions, as: Conditions + alias Archethic.Contracts.ContractConstants, as: Constants alias Archethic.Contracts.Interpreter @@ -14,7 +13,7 @@ defmodule Archethic.Contracts.Contract do alias Archethic.TransactionChain.Transaction alias Archethic.TransactionChain.TransactionData - defstruct triggers: [], + defstruct triggers: %{}, conditions: %{ transaction: %Conditions{}, inherit: %Conditions{}, @@ -23,12 +22,15 @@ defmodule Archethic.Contracts.Contract do constants: %Constants{}, next_transaction: %Transaction{data: %TransactionData{}} - @type trigger_type() :: :datetime | :interval | :transaction + @type trigger_type() :: + {:datetime, DateTime.t()} | {:interval, String.t()} | :transaction | :oracle @type condition() :: :transaction | :inherit | :oracle @type origin_family :: SharedSecrets.origin_family() @type t() :: %__MODULE__{ - triggers: list(Trigger.t()), + triggers: %{ + trigger_type() => Macro.t() + }, conditions: %{ transaction: Conditions.t(), inherit: Conditions.t(), @@ -55,48 +57,24 @@ defmodule Archethic.Contracts.Contract do @doc """ Add a trigger to the contract """ - @spec add_trigger(t(), Trigger.type(), Keyword.t(), Macro.t()) :: t() + @spec add_trigger(map(), trigger_type(), Macro.t()) :: t() def add_trigger( contract = %__MODULE__{}, - :datetime, - opts = [at: _datetime = %DateTime{}], + type, actions ) do - do_add_trigger(contract, %Trigger{type: :datetime, opts: opts, actions: actions}) - end - - def add_trigger( - contract = %__MODULE__{}, - :interval, - opts = [at: interval], - actions - ) - when is_binary(interval) do - do_add_trigger(contract, %Trigger{type: :interval, opts: opts, actions: actions}) - end - - def add_trigger(contract = %__MODULE__{}, :transaction, _, actions) do - do_add_trigger(contract, %Trigger{type: :transaction, actions: actions}) - end - - def add_trigger(contract = %__MODULE__{}, :oracle, _, actions) do - do_add_trigger(contract, %Trigger{type: :oracle, actions: actions}) - end - - defp do_add_trigger(contract, trigger = %Trigger{}) do - Map.update!(contract, :triggers, &(&1 ++ [trigger])) + Map.update!(contract, :triggers, &Map.put(&1, type, actions)) end @doc """ Add a condition to the contract """ - @spec add_condition(t(), condition(), any()) :: t() + @spec add_condition(map(), condition(), Conditions.t()) :: t() def add_condition( - contract = %__MODULE__{conditions: conditions}, + contract = %__MODULE__{}, condition_name, - condition - ) - when condition_name in [:transaction, :inherit, :oracle] do - %{contract | conditions: Map.put(conditions, condition_name, condition)} + conditions = %Conditions{} + ) do + Map.update!(contract, :conditions, &Map.put(&1, condition_name, conditions)) end end diff --git a/lib/archethic/contracts/contract/conditions.ex b/lib/archethic/contracts/contract/conditions.ex index fb58bb95d4..81cbae15d9 100644 --- a/lib/archethic/contracts/contract/conditions.ex +++ b/lib/archethic/contracts/contract/conditions.ex @@ -1,9 +1,10 @@ -defmodule Archethic.Contracts.Contract.Conditions do +defmodule Archethic.Contracts.ContractConditions do @moduledoc """ Represents the smart contract conditions """ defstruct [ + :address, :type, :content, :code, @@ -20,6 +21,7 @@ defmodule Archethic.Contracts.Contract.Conditions do alias Archethic.TransactionChain.Transaction @type t :: %__MODULE__{ + address: binary() | Macro.t() | nil, type: Transaction.transaction_type() | nil, content: binary() | Macro.t() | nil, code: binary() | Macro.t() | nil, @@ -33,6 +35,7 @@ defmodule Archethic.Contracts.Contract.Conditions do } def empty?(%__MODULE__{ + address: nil, type: nil, content: nil, code: nil, diff --git a/lib/archethic/contracts/contract/constants.ex b/lib/archethic/contracts/contract/constants.ex index 4177a65706..229a4a2bce 100644 --- a/lib/archethic/contracts/contract/constants.ex +++ b/lib/archethic/contracts/contract/constants.ex @@ -1,4 +1,4 @@ -defmodule Archethic.Contracts.Contract.Constants do +defmodule Archethic.Contracts.ContractConstants do @moduledoc """ Represents the smart contract constants and bindings """ @@ -6,7 +6,7 @@ defmodule Archethic.Contracts.Contract.Constants do defstruct [:contract, :transaction] @type t :: %__MODULE__{ - contract: map(), + contract: map() | nil, transaction: map() | nil } @@ -153,4 +153,67 @@ defmodule Archethic.Contracts.Contract.Constants do previous_public_key: Map.get(constants, "previous_public_key") } end + + @doc """ + Stringify binary transaction values + """ + @spec stringify(map()) :: map() + def stringify(constants = %{}) do + %{ + "address" => apply_not_nil(constants, "address", &Base.encode16/1), + "type" => Map.get(constants, "type"), + "content" => Map.get(constants, "content"), + "code" => Map.get(constants, "code"), + "authorized_keys" => + apply_not_nil(constants, "authorized_keys", fn authorized_keys -> + authorized_keys + |> Enum.map(fn {public_key, encrypted_secret_key} -> + {Base.encode16(public_key), Base.encode16(encrypted_secret_key)} + end) + |> Enum.into(%{}) + end), + "authorized_public_keys" => + apply_not_nil(constants, "authorized_public_keys", fn public_keys -> + Enum.map(public_keys, &Base.encode16/1) + end), + "secrets" => + apply_not_nil(constants, "secrets", fn secrets -> + Enum.map(secrets, &Base.encode16/1) + end), + "previous_public_key" => apply_not_nil(constants, "previous_public_key", &Base.encode16/1), + "recipients" => + apply_not_nil(constants, "recipients", fn recipients -> + Enum.map(recipients, &Base.encode16/1) + end), + "uco_transfers" => + apply_not_nil(constants, "uco_transfers", fn transfers -> + transfers + |> Enum.map(fn {to, amount} -> + {Base.encode16(to), amount} + end) + |> Enum.into(%{}) + end), + "token_transfers" => + apply_not_nil(constants, "token_transfers", fn transfers -> + transfers + |> Enum.map(fn {to, transfers} -> + {Base.encode16(to), + Enum.map(transfers, fn transfer -> + Map.update!(transfer, "token_address", &Base.encode16/1) + end)} + end) + end), + "timestamp" => Map.get(constants, "timestamp") + } + end + + defp apply_not_nil(map, key, fun) do + case Map.get(map, key) do + nil -> + nil + + val -> + fun.(val) + end + end end diff --git a/lib/archethic/contracts/contract/trigger.ex b/lib/archethic/contracts/contract/trigger.ex deleted file mode 100644 index 697268c09b..0000000000 --- a/lib/archethic/contracts/contract/trigger.ex +++ /dev/null @@ -1,19 +0,0 @@ -defmodule Archethic.Contracts.Contract.Trigger do - @moduledoc """ - Represents the smart contract triggers - """ - - defstruct [:type, :actions, opts: []] - - @type timestamp :: non_neg_integer() - @type interval :: binary() - @type address :: binary() - - @type type() :: :datetime | :interval | :transaction | :oracle - - @type t :: %__MODULE__{ - type: type(), - opts: Keyword.t(), - actions: Macro.t() - } -end diff --git a/lib/archethic/contracts/interpreter.ex b/lib/archethic/contracts/interpreter.ex index b5c9c69ddc..f09c8df621 100644 --- a/lib/archethic/contracts/interpreter.ex +++ b/lib/archethic/contracts/interpreter.ex @@ -3,51 +3,12 @@ defmodule Archethic.Contracts.Interpreter do require Logger - alias Crontab.CronExpression.Parser, as: CronParser + alias Archethic.Contracts.ActionInterpreter + alias Archethic.Contracts.ConditionInterpreter - alias __MODULE__.Library - alias __MODULE__.TransactionStatements + alias __MODULE__.Utils, as: InterpreterUtils alias Archethic.Contracts.Contract - alias Archethic.Contracts.Contract.Conditions - - alias Archethic.SharedSecrets - - alias Archethic.TransactionChain.Transaction - alias Archethic.TransactionChain.TransactionData - - @library_functions_names Library.__info__(:functions) - |> Enum.map(&Atom.to_string(elem(&1, 0))) - - @transaction_statements_functions_names TransactionStatements.__info__(:functions) - |> Enum.map(&Atom.to_string(elem(&1, 0))) - - @transaction_fields [ - "address", - "type", - "timestamp", - "previous_signature", - "previous_public_key", - "origin_signature", - "content", - "keys", - "code", - "uco_ledger", - "token_ledger", - "uco_transfers", - "token_transfers", - "authorized_public_keys", - "secrets", - "recipients" - ] - - @condition_fields Conditions.__struct__() - |> Map.keys() - |> Enum.reject(&(&1 == :__struct__)) - |> Enum.map(&Atom.to_string/1) - - @transaction_types Transaction.types() |> Enum.map(&Atom.to_string/1) - @origin_families SharedSecrets.list_origin_families() |> Enum.map(&Atom.to_string/1) @doc ~S""" Parse a smart contract code and return the filtered AST representation. @@ -84,7 +45,7 @@ defmodule Archethic.Contracts.Interpreter do {:ok, %Contract{ conditions: %{ - inherit: %Archethic.Contracts.Contract.Conditions{ + inherit: %Archethic.Contracts.ContractConditions{ content: { :==, [line: 7], @@ -101,7 +62,7 @@ defmodule Archethic.Contracts.Interpreter do type: nil, uco_transfers: nil }, - oracle: %Archethic.Contracts.Contract.Conditions{ + oracle: %Archethic.Contracts.ContractConditions{ content: { :>, [line: 11], @@ -118,14 +79,14 @@ defmodule Archethic.Contracts.Interpreter do ] }, authorized_keys: nil, - code: nil, + code: nil, token_transfers: nil, origin_family: :all, previous_public_key: nil, type: nil, uco_transfers: nil }, - transaction: %Archethic.Contracts.Contract.Conditions{ + transaction: %Archethic.Contracts.ContractConditions{ content: { :==, [line: 2], @@ -143,11 +104,10 @@ defmodule Archethic.Contracts.Interpreter do uco_transfers: nil } }, - constants: %Constants{}, + constants: %Archethic.Contracts.ContractConstants{}, next_transaction: %Transaction{ data: %TransactionData{}}, - triggers: [ - %Trigger{ - actions: {:__block__, [], + triggers: %{ + {:datetime, ~U[2020-10-21 08:56:43Z]} => {:__block__, [], [ {:=, [line: 15], [{:scope, [line: 15], nil}, {{:., [line: 15], [{:__aliases__, [line: 15], [:Map]}, :put]}, [line: 15], [{:scope, [line: 15], nil}, "new_content", "Sent 1040000000"]}]}, { @@ -171,78 +131,57 @@ defmodule Archethic.Contracts.Interpreter do [line: 18], [ {:scope, [line: 18], nil}, - {:update_in, [line: 18], [{:scope, [line: 18], nil}, ["next_transaction"], {:&, [line: 18], [{{:., [line: 18], [{:__aliases__, [alias: Archethic.Contracts.Interpreter.TransactionStatements], [:TransactionStatements]}, :add_uco_transfer]}, [line: 18], [{:&, [line: 18], [1]}, [{"to", <<34, 54, 139, 80, 211, 178, 151, 103, 135, 207, 204, 39, 80, 138, 142, 140, 103, 72, 50, 25, 130, 95, 153, 143, 201, 214, 144, 141, 84, 208, 254, 16>>}, {"amount", 1040000000}]]}]}]} + {:update_in, [line: 18], [{:scope, [line: 18], nil}, ["next_transaction"], {:&, [line: 18], [{{:., [line: 18], [{:__aliases__, [alias: Archethic.Contracts.Interpreter.TransactionStatements], [:TransactionStatements]}, :add_uco_transfer]}, [line: 18], [{:&, [line: 18], [1]}, [{"to", "22368B50D3B2976787CFCC27508A8E8C67483219825F998FC9D6908D54D0FE10"}, {"amount", 1040000000}]]}]}]} ] } ]}, - opts: [at: ~U[2020-10-21 08:56:43Z]], - type: :datetime - }, - %Trigger{actions: { + :oracle => { :=, [line: 22], [ {:scope, [line: 22], nil}, {:update_in, [line: 22], [{:scope, [line: 22], nil}, ["next_transaction"], {:&, [line: 22], [{{:., [line: 22], [{:__aliases__, [alias: Archethic.Contracts.Interpreter.TransactionStatements], [:TransactionStatements]}, :set_content]}, [line: 22], [{:&, [line: 22], [1]}, "uco price changed"]}]}]} ] - }, opts: [], type: :oracle} - ] + } + } } } - # Returns an error when there are invalid trigger options + Returns an error when there are invalid trigger options - # iex> Interpreter.parse(" - # ...> actions triggered_by: datetime, at: 0000000 do - # ...> end - # ...> ") - # {:error, "invalid trigger - invalid datetime - arguments at:0 - L1"} + iex> Interpreter.parse(" + ...> actions triggered_by: datetime, at: 0000000 do + ...> end + ...> ") + {:error, "invalid datetime's trigger"} - # Returns an error when a invalid term is provided + Returns an error when a invalid term is provided - # iex> Interpreter.parse(" - # ...> actions do - # ...> System.user_home - # ...> end - # ...> ") - # {:error, "unexpected token - System - L2"} + iex> Interpreter.parse(" + ...> actions triggered_by: transaction do + ...> System.user_home + ...> end + ...> ") + {:error, "unexpected term - System - L2"} """ @spec parse(code :: binary()) :: {:ok, Contract.t()} | {:error, reason :: binary()} def parse(code) when is_binary(code) do - with {:ok, ast} <- - Code.string_to_quoted(String.trim(code), - static_atoms_encoder: &atom_encoder/2 - ), - {_, {:ok, %{contract: contract}}} <- - Macro.traverse( - ast, - {:ok, %{scope: :root, contract: %Contract{}}}, - &prewalk/2, - &postwalk/2 - ) do + with {:ok, ast} <- sanitize_code(code), + {:ok, contract} <- parse_contract(ast, %Contract{}) do {:ok, contract} else - {_node, {:error, reason}} -> - {:error, format_error_reason(reason)} + {:error, {meta, {_, info}, token}} -> + {:error, InterpreterUtils.format_error_reason({token, meta, []}, info)} - {:error, reason} -> - {:error, format_error_reason(reason)} - end - catch - {{:error, reason}, {:., metadata, [{:__aliases__, _, atom: cause}, _]}} -> - {:error, format_error_reason({metadata, reason, cause})} - - {{:error, :unexpected_token}, {:atom, key}} -> - {:error, format_error_reason({[], "unexpected_token", key})} - - {{:error, :unexpected_token}, {{:atom, key}, metadata, _}} -> - {:error, format_error_reason({metadata, "unexpected_token", key})} + {:error, {meta, info, token}} -> + {:error, InterpreterUtils.format_error_reason({token, meta, []}, info)} - {{:error, :unexpected_token}, {{:atom, key}, _}} -> - {:error, format_error_reason({[], "unexpected_token", key})} + {:error, {:unexpected_term, ast}} -> + {:error, InterpreterUtils.format_error_reason(ast, "unexpected term")} - {:error, reason = {_metadata, _message, _cause}} -> - {:error, format_error_reason(reason)} + {:error, reason} -> + {:error, reason} + end end defp atom_encoder(atom, _) do @@ -253,1244 +192,52 @@ defmodule Archethic.Contracts.Interpreter do end end - defp format_error_reason({metadata, message, cause}) do - message = prepare_message(message) - - [prepare_message(message), cause, metadata_to_string(metadata)] - |> Enum.reject(&(&1 == "")) - |> Enum.join(" - ") - end - - defp prepare_message(message) when is_atom(message) do - message |> Atom.to_string() |> String.replace("_", " ") - end - - defp prepare_message(message) when is_binary(message) do - String.trim_trailing(message, ":") - end - - defp metadata_to_string(line: line, column: column), do: "L#{line}:C#{column}" - defp metadata_to_string(line: line), do: "L#{line}" - defp metadata_to_string(_), do: "" - - # Whitelist operators - defp prewalk(node = {:+, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, - do: {node, acc} - - defp prewalk(node = {:-, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, - do: {node, acc} - - defp prewalk(node = {:/, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, - do: {node, acc} - - defp prewalk(node = {:*, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, - do: {node, acc} - - defp prewalk(node = {:>, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, - do: {node, acc} - - defp prewalk(node = {:<, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, - do: {node, acc} - - defp prewalk(node = {:>=, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, - do: {node, acc} - - defp prewalk(node = {:<=, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, - do: {node, acc} - - defp prewalk(node = {:|>, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, - do: {node, acc} - - defp prewalk(node = {:==, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, - do: {node, acc} - - # Allow variable assignation inside the actions - defp prewalk(node = {:=, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, - do: {node, acc} - - # Allow variable as function argument - defp prewalk(node = {{:atom, _}, {{:atom, _varname}, _, _}}, acc = {:ok, %{scope: scope}}) - when scope != :root do - {node, acc} - end - - # Whitelist the use of doted statement - defp prewalk(node = {{:., _, [{_, _, _}, _]}, _, []}, acc = {:ok, %{scope: scope}}) - when scope != :root, - do: {node, acc} - - # # Whitelist the definition of globals in the root - # defp prewalk(node = {:@, _, [{key, _, [val]}]}, acc = {:ok, :root}) - # when is_atom(key) and not is_nil(val), - # do: {node, acc} - - # # Whitelist the use of globals - # defp prewalk(node = {:@, _, [{key, _, nil}]}, acc = {:ok, _}) when is_atom(key), - # do: {node, acc} - - # Whitelist conditional oeprators - defp prewalk(node = {:if, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, - do: {node, acc} - - defp prewalk(node = {:else, _}, acc = {:ok, %{scope: scope}}) when scope != :root, - do: {node, acc} - - defp prewalk(node = [do: _, else: _], acc = {:ok, %{scope: scope}}) when scope != :root, - do: {node, acc} - - defp prewalk(node = :else, acc = {:ok, %{scope: scope}}) when scope != :root, do: {node, acc} - - defp prewalk(node = {:and, _, _}, acc = {:ok, _}), do: {node, acc} - defp prewalk(node = {:or, _, _}, acc = {:ok, _}), do: {node, acc} - - # Whitelist the in operation - defp prewalk(node = {:in, _, [_, _]}, acc = {:ok, _}), do: {node, acc} - - # Whitelist maps - defp prewalk(node = {:%{}, _, fields}, acc = {:ok, _}) when is_list(fields), do: {node, acc} - - defp prewalk(node = {key, _val}, acc) when is_binary(key) do - {node, acc} - end - - # Whitelist the multiline - defp prewalk(node = {{:__block__, _, _}}, acc = {:ok, _}) do - {node, acc} - end - - defp prewalk(node = {:__block__, _, _}, acc = {:ok, _}) do - {node, acc} - end - - # Whitelist custom atom - defp prewalk(node = :atom, acc), do: {node, acc} - - # Whitelist the actions DSL - defp prewalk(node = {{:atom, "actions"}, _, _}, {:ok, acc = %{scope: :root}}) do - {node, {:ok, %{acc | scope: :actions}}} - end - - # Whitelist the triggered_by DSL in the actions - defp prewalk( - node = [ - {{:atom, "triggered_by"}, {{:atom, trigger_type}, meta = [line: _], _}} | trigger_opts - ], - acc = {:ok, %{scope: :actions}} - ) - when trigger_type in ["datetime", "interval", "transaction", "oracle"] do - case valid_trigger_opts(trigger_type, trigger_opts) do - :ok -> - {node, acc} - - {:error, reason} -> - params = Enum.map_join(trigger_opts, ", ", fn {{:atom, k}, v} -> "#{k}:#{v}" end) - - {node, {:error, {meta, "invalid trigger - #{reason}", "arguments #{params}"}}} - end - end - - defp prewalk( - node = {{:atom, "triggered_by"}, {{:atom, trigger_type}, _, _}}, - acc = {:ok, %{scope: :actions}} - ) - when trigger_type in ["datetime", "interval", "transaction", "oracle"], - do: {node, acc} - - defp prewalk(node = {:atom, trigger_type}, acc = {:ok, %{scope: :actions}}) - when trigger_type in ["datetime", "interval", "transaction", "oracle"], - do: {node, acc} - - # Whitelist triggers 'at' argument - defp prewalk(node = {{:atom, "at"}, _arg}, acc = {:ok, %{scope: :actions}}), do: {node, acc} - - # Whitelist the condition DSL - defp prewalk( - node = {{:atom, "condition"}, _, [{{:atom, condition_name}, [_]}]}, - {:ok, acc = %{scope: :root}} - ) - when condition_name in ["inherit", "transaction", "oracle"] do - {node, {:ok, %{acc | scope: :condition}}} - end - - defp prewalk(node = [{{:atom, condition}, [_ | _]}], acc = {:ok, %{scope: :condition}}) - when condition in ["inherit", "transaction", "oracle"] do - {node, acc} - end - - defp prewalk( - node = {{:atom, "condition"}, _, [[{{:atom, condition_name}, _}]]}, - {:ok, acc = %{scope: :root}} - ) - when condition_name in ["inherit", "transaction", "oracle"] do - {node, {:ok, %{acc | scope: :condition}}} - end - - defp prewalk(node = [{{:atom, _}, _} | _], acc = {:ok, %{scope: :condition}}) do - {node, acc} - end - - defp prewalk(node = {{:atom, condition_name}, _}, acc = {:ok, %{scope: :condition}}) - when condition_name in ["inherit", "transaction", "oracle"] do - {node, acc} - end - - defp prewalk(node = {:atom, condition_name}, acc = {:ok, %{scope: :condition}}) - when condition_name in ["inherit", "transaction", "oracle"], - do: {node, acc} - - defp prewalk(node = {:atom, field_name}, acc = {:ok, %{scope: :condition}}) - when field_name in @condition_fields, - do: {node, acc} - - # Whitelist the usage of map in the conditions - # Example: - # - # ``` - # condition inehrit: [ - # uco_transfers: [%{ to: "address", amount: 1000000 }] - # ] - # ``` - defp prewalk(node = {{:atom, _key}, _val}, acc = {:ok, %{scope: :condition}}) do - {node, acc} - end - - # Whitelist the usage transaction fields - defp prewalk( - node = {:., _, [{{:atom, transaction_ref}, _, nil}, {:atom, type}]}, - acc - ) - when transaction_ref in ["next", "previous", "transaction", "contract"] and - type in @transaction_fields do - {node, acc} - end - - defp prewalk( - node = - {{:atom, _}, {{:., _, [{{:atom, transaction_ref}, _, nil}, {:atom, type}]}, _, _}}, - acc - ) - when transaction_ref in ["next", "previous", "transaction", "contract"] and - type in @transaction_fields do - {node, acc} - end - - # Whitelist the use of list - defp prewalk(node = [{{:atom, _}, _, nil} | _], acc = {:ok, %{scope: scope}}) - when scope != :root do - {node, acc} - end - - # Whitelist access to map field - defp prewalk( - node = {{:., _, [Access, :get]}, _, _}, - acc = {:ok, %{scope: scope}} - ) - when scope != :root do - {node, acc} - end - - defp prewalk(node = {:., _, [Access, :get]}, acc = {:ok, %{scope: scope}}) when scope != :root, - do: {node, acc} - - defp prewalk(node = Access, acc), do: {node, acc} - defp prewalk(node = :get, acc), do: {node, acc} - - # Whitelist condition fields - - # Whitelist the origin family condition - defp prewalk( - node = {{:atom, "origin_family"}, {{:atom, family}, metadata, nil}}, - acc = {:ok, %{scope: :condition}} - ) do - if family in @origin_families do - {node, acc} - else - {node, {:error, metadata, "unexpected token"}, "invalid origin family"} - end - end - - defp prewalk(node = {:atom, origin_family}, acc = {:ok, %{scope: :condition}}) - when origin_family in @origin_families, - do: {node, acc} - - # Whitelist the transaction type condition - defp prewalk( - node = {{:atom, "type"}, {{:atom, type}, metadata, nil}}, - acc = {:ok, %{scope: :condition}} - ) do - if type in @transaction_types do - {node, acc} - else - {node, {:error, metadata, "unexpected token"}, "invalid transaction type"} - end - end - - defp prewalk( - node = {{:atom, field}, _}, - acc = {:ok, %{scope: :condition}} - ) - when field in @condition_fields do - {node, acc} - end - - defp prewalk( - node = {{:atom, field}, [line: _], _}, - acc = {:ok, %{scope: {:function, _}}} - ) - when field in @transaction_fields do - {node, acc} - end - - # Whitelist the size/1 function - defp prewalk( - node = {{:atom, "size"}, _, [_data]}, - acc = {:ok, %{scope: scope}} - ) - when scope != :root do - {node, acc} - end - - # Whitelist the hash/1 function - defp prewalk(node = {{:atom, "hash"}, _, [_data]}, acc = {:ok, %{scope: scope}}) - when scope != :root, - do: {node, acc} - - # Whitelist the regex_match?/2 function - defp prewalk( - node = {{:atom, "regex_match?"}, _, [_input, _search]}, - acc = {:ok, %{scope: scope}} - ) - when scope != :root, - do: {node, acc} - - # Whitelist the regex_extract/2 function - defp prewalk( - node = {{:atom, "regex_extract"}, _, [_input, _search]}, - acc = {:ok, %{scope: scope}} - ) - when scope != :root, - do: {node, acc} - - # Whitelist the json_path_extract/2 function - defp prewalk( - node = {{:atom, "json_path_extract"}, _, [_input, _search]}, - acc = {:ok, %{scope: scope}} - ) - when scope != :root, - do: {node, acc} - - # Whitelist the json_path_match?/2 function - defp prewalk( - node = {{:atom, "json_path_match?"}, _, [_input, _search]}, - acc = {:ok, %{scope: scope}} - ) - when scope != :root, - do: {node, acc} - - # Whitelist the get_genesis_address/1 function - defp prewalk( - node = {{:atom, "get_genesis_address"}, _, [_address]}, - acc = {:ok, %{scope: scope}} - ) - when scope != :root do - {node, acc} - end - - # Whitelist the timestamp/0 function - defp prewalk(node = {{:atom, "timestamp"}, _, []}, acc = {:ok, %{scope: scope}}) - when scope != :root, - do: {node, acc} - - defp prewalk( - node = {{:atom, "get_genesis_public_key"}, _, [_address]}, - acc = {:ok, %{scope: scope}} - ) - when scope != :root do - {node, acc} - end - - # Whitelist the regex_match?/1 function in the condition - defp prewalk( - node = {{:atom, "regex_match?"}, _, [_search]}, - acc = {:ok, %{scope: :condition}} - ) do - {node, acc} - end - - # Whitelist the json_path_extract/1 function in the condition - defp prewalk( - node = {{:atom, "json_path_extract"}, _, [_search]}, - acc = {:ok, %{scope: :condition}} - ) do - {node, acc} - end - - # Whitelist the json_path_match?/1 function in the condition - defp prewalk( - node = {{:atom, "json_path_match?"}, _, [_search]}, - acc = {:ok, %{scope: :condition}} - ) do - {node, acc} - end - - # Whitelist the hash/0 function in the condition - defp prewalk( - node = {{:atom, "hash"}, _, []}, - acc = {:ok, %{scope: :condition}} - ) do - {node, acc} - end - - # Whitelist the in?/1 function in the condition - defp prewalk( - node = {{:atom, "in?"}, _, [_data]}, - acc = {:ok, %{scope: :condition}} - ) do - {node, acc} - end - - # Whitelist the size/0 function in the condition - defp prewalk(node = {{:atom, "size"}, _, []}, acc = {:ok, %{scope: :condition}}), - do: {node, acc} - - # Whitelist the get_genesis_address/0 function in condition - defp prewalk( - node = {{:atom, "get_genesis_address"}, _, []}, - acc = {:ok, %{scope: :condition}} - ) do - {node, acc} - end - - # Whitelist the get_genesis_public_key/0 function in condition - defp prewalk( - node = {{:atom, "get_genesis_public_key"}, _, []}, - acc = {:ok, %{scope: :condition}} - ) do - {node, acc} - end - - # Whitelist the timestamp/0 function in condition - defp prewalk(node = {{:atom, "timestamp"}, _, []}, acc = {:ok, %{scope: :condition}}), - do: {node, acc} - - # Whitelist the used of functions in the actions - defp prewalk(node = {{:atom, fun_name}, _, _}, {:ok, acc = %{scope: :actions}}) - when fun_name in @transaction_statements_functions_names, - do: {node, {:ok, %{acc | scope: {:function, fun_name, :actions}}}} - - defp prewalk( - node = [{{:atom, _variable_name}, _, nil}], - acc = {:ok, %{scope: {:function, "set_content", :actions}}} - ) do - {node, acc} - end - - # Whitelist the add_uco_transfer argument list - defp prewalk( - node = [{{:atom, "to"}, _to}, {{:atom, "amount"}, _amount}], - acc = {:ok, %{scope: {:function, "add_uco_transfer", :actions}}} - ) do - {node, acc} - end - - defp prewalk( - node = {{:atom, arg}, _}, - acc = {:ok, %{scope: {:function, "add_uco_transfer", :actions}}} - ) - when arg in ["to", "amount"], - do: {node, acc} - - # Whitelist the add_token_tranfser argument list - defp prewalk( - node = [ - {{:atom, "to"}, _to}, - {{:atom, "amount"}, _amount}, - {{:atom, "token_address"}, _token_address}, - {{:atom, "token_id"}, _token_id} - ], - acc = {:ok, %{scope: {:function, "add_token_transfer", :actions}}} - ) do - {node, acc} - end - - defp prewalk( - node = {{:atom, arg}, _}, - acc = {:ok, %{scope: {:function, "add_token_transfer", :actions}}} - ) - when arg in ["to", "amount", "token_address", "token_id"], - do: {node, acc} - - # Whitelist the add_ownership argument list - defp prewalk( - node = [ - {{:atom, "secret"}, _secret}, - {{:atom, "secret_key"}, _secret_key}, - {{:atom, "authorized_public_keys"}, _authorized_public_keys} - ], - acc = {:ok, %{scope: {:function, "add_ownership", :actions}}} - ) do - {node, acc} - end - - defp prewalk( - node = {{:atom, arg}, _}, - acc = {:ok, %{scope: {:function, "add_ownership", :actions}}} - ) - when arg in ["secret", "secret_key", "authorized_public_keys"], - do: {node, acc} - - # Whitelist generics - defp prewalk(true, acc = {:ok, _}), do: {true, acc} - defp prewalk(false, acc = {:ok, _}), do: {false, acc} - defp prewalk(number, acc = {:ok, _}) when is_number(number), do: {number, acc} - defp prewalk(string, acc = {:ok, _}) when is_binary(string), do: {string, acc} - defp prewalk(node = [do: _], acc = {:ok, _}), do: {node, acc} - defp prewalk(node = {:do, _}, acc = {:ok, _}), do: {node, acc} - defp prewalk(node = :do, acc = {:ok, _}), do: {node, acc} - # Literals - defp prewalk(node = {{:atom, key}, _, nil}, acc = {:ok, _}) when is_binary(key), - do: {node, acc} - - defp prewalk(node = {:atom, key}, acc = {:ok, _}) when is_binary(key), do: {node, acc} - - # Whitelist interpolation of strings - defp prewalk( - node = - {:<<>>, _, [{:"::", _, [{{:., _, [Kernel, :to_string]}, _, _}, {:binary, _, nil}]}, _]}, - acc - ) do - {node, acc} - end - - defp prewalk( - node = - {:<<>>, _, - [ - _, - {:"::", _, [{{:., _, [Kernel, :to_string]}, _, _}, _]} - ]}, - acc - ) do - {node, acc} - end - - defp prewalk(node = {:"::", _, [{{:., _, [Kernel, :to_string]}, _, _}, _]}, acc) do - {node, acc} - end - - defp prewalk(node = {{:., _, [Kernel, :to_string]}, _, _}, acc) do - {node, acc} - end - - defp prewalk(node = {:., _, [Kernel, :to_string]}, acc) do - {node, acc} - end - - defp prewalk(node = Kernel, acc), do: {node, acc} - defp prewalk(node = :to_string, acc), do: {node, acc} - defp prewalk(node = {:binary, _, nil}, acc), do: {node, acc} - - defp prewalk(node, acc) when is_list(node) do - {node, acc} - end - - # Blacklist anything else - defp prewalk(node, {:ok, _acc}) do - throw({{:error, :unexpected_token}, node}) - end - - defp prewalk(node, e = {:error, _}), do: {node, e} - - # Reset the scope after actions triggered block ending - defp postwalk( - node = - {{:atom, "actions"}, [line: _], - [[{{:atom, "triggered_by"}, {{:atom, trigger_type}, _, _}} | opts], [do: actions]]}, - {:ok, acc} - ) do - actions = - inject_bindings_and_functions(actions, - bindings: %{ - "contract" => Enum.map(@transaction_fields, &{&1, ""}) |> Enum.into(%{}), - "transaction" => Enum.map(@transaction_fields, &{&1, ""}) |> Enum.into(%{}) - } - ) - - acc = - case trigger_type do - "datetime" -> - [{{:atom, "at"}, timestamp}] = opts - datetime = DateTime.from_unix!(timestamp) - - Map.update!( - acc, - :contract, - &Contract.add_trigger(&1, :datetime, [at: datetime], actions) - ) - - "interval" -> - [{{:atom, "at"}, interval}] = opts - - Map.update!( - acc, - :contract, - &Contract.add_trigger( - &1, - :interval, - [at: interval], - actions - ) - ) - - "transaction" -> - Map.update!(acc, :contract, &Contract.add_trigger(&1, :transaction, [], actions)) - - "oracle" -> - Map.update!(acc, :contract, &Contract.add_trigger(&1, :oracle, [], actions)) - end - - {node, {:ok, %{acc | scope: :root}}} - end - - # Add conditions with brackets - defp postwalk( - node = {{:atom, "condition"}, _, [[{{:atom, condition_name}, conditions}]]}, - {:ok, acc} - ) do - bindings = Enum.map(@transaction_fields, &{&1, ""}) |> Enum.into(%{}) - - bindings = - case condition_name do - "inherit" -> - Map.merge(bindings, %{ - "next" => Enum.map(@transaction_fields, &{&1, ""}) |> Enum.into(%{}), - "previous" => Enum.map(@transaction_fields, &{&1, ""}) |> Enum.into(%{}) - }) - - _ -> - Map.merge(bindings, %{ - "contract" => Enum.map(@transaction_fields, &{&1, ""}) |> Enum.into(%{}), - "transaction" => Enum.map(@transaction_fields, &{&1, ""}) |> Enum.into(%{}) - }) - end - - subject_scope = if condition_name == "inherit", do: "next", else: "transaction" - - conditions = - inject_bindings_and_functions(conditions, - bindings: bindings, - subject: subject_scope - ) - - new_acc = - acc - |> Map.update!( - :contract, - &Contract.add_condition( - &1, - String.to_existing_atom(condition_name), - aggregate_conditions(conditions, subject_scope) - ) - ) - |> Map.put(:scope, :root) - - {node, {:ok, new_acc}} - end - - # Add complex conditions with if statements - defp postwalk( - node = - {{:atom, "condition"}, _, - [ - {{:atom, condition_name}, _, - [ - conditions - ]} - ]}, - {:ok, acc} - ) do - subject_scope = if condition_name == "inherit", do: "next", else: "transaction" - - bindings = Enum.map(@transaction_fields, &{&1, ""}) |> Enum.into(%{}) - - bindings = - case condition_name do - "inherit" -> - Map.merge(bindings, %{ - "next" => Enum.map(@transaction_fields, &{&1, ""}) |> Enum.into(%{}), - "previous" => Enum.map(@transaction_fields, &{&1, ""}) |> Enum.into(%{}) - }) - - _ -> - Map.merge(bindings, %{ - "contract" => Enum.map(@transaction_fields, &{&1, ""}) |> Enum.into(%{}), - "transaction" => Enum.map(@transaction_fields, &{&1, ""}) |> Enum.into(%{}) - }) - end - - conditions = - inject_bindings_and_functions(conditions, - bindings: bindings, - subject: subject_scope - ) - - new_acc = - acc - |> Map.update!( - :contract, - &Contract.add_condition( - &1, - String.to_existing_atom(condition_name), - aggregate_conditions(conditions, subject_scope) - ) - ) - |> Map.put(:scope, :root) - - {node, {:ok, new_acc}} - end - - # Return to the parent scope after parsing the function call - defp postwalk(node = {{:atom, _}, _, _}, {:ok, acc = %{scope: {:function, _, scope}}}) do - {node, {:ok, %{acc | scope: scope}}} - end - - # Convert Access key string to binary - defp postwalk( - {{:., meta1, [Access, :get]}, meta2, - [{{:., meta3, [{subject, meta4, nil}, {:atom, field}]}, meta5, []}, {:atom, key}]}, - acc = {:ok, _} - ) do - { - {{:., meta1, [Access, :get]}, meta2, - [ - {{:., meta3, [{subject, meta4, nil}, String.to_existing_atom(field)]}, meta5, []}, - Base.decode16!(key, case: :mixed) - ]}, - acc - } - end - - # Convert map key to binary - defp postwalk({:%{}, meta, params}, acc = {:ok, _}) do - encoded_params = - Enum.map(params, fn - {{:atom, key}, value} when is_binary(key) -> - case Base.decode16(key, case: :mixed) do - {:ok, bin} -> - {bin, value} - - :error -> - {key, value} - end - - {key, value} -> - {key, value} - end) - - {{:%{}, meta, encoded_params}, acc} - end - - defp postwalk(node, acc), do: {node, acc} - - defp valid_trigger_opts("datetime", [{{:atom, "at"}, timestamp}]) do - if length(Integer.digits(timestamp)) != 10 do - {:error, "invalid datetime"} - else - case DateTime.from_unix(timestamp) do - {:ok, _} -> - :ok - - _ -> - {:error, "invalid datetime"} - end - end - end - - defp valid_trigger_opts("interval", [{{:atom, "at"}, interval}]) do - case CronParser.parse(interval) do - {:ok, _} -> - :ok - - {:error, _} -> - {:error, "invalid interval"} - end - end - - defp valid_trigger_opts("transaction", []), do: :ok - defp valid_trigger_opts("oracle", []), do: :ok - - defp valid_trigger_opts(_, _), do: {:error, "unexpected token"} - - defp aggregate_conditions(conditions, subject_scope) do - Enum.reduce(conditions, %Conditions{}, fn {subject, condition}, acc -> - condition = do_aggregate_condition(condition, subject_scope, subject) - Map.put(acc, String.to_existing_atom(subject), condition) - end) - end - - defp do_aggregate_condition(condition, _, "origin_family"), - do: String.to_existing_atom(condition) - - defp do_aggregate_condition(condition, subject_scope, subject) - when is_binary(condition) or is_number(condition) do - {:==, [], - [ - {:get_in, [], [{:scope, [], nil}, [subject_scope, subject]]}, - condition - ]} + @spec sanitize_code(binary()) :: {:ok, Macro.t()} | {:error, any()} + def sanitize_code(code) when is_binary(code) do + code + |> String.trim() + |> Code.string_to_quoted(static_atoms_encoder: &atom_encoder/2) end - defp do_aggregate_condition(condition, subject_scope, subject) when is_list(condition) do - {:==, [], - [ - {:get_in, [], - [ - {:scope, [], nil}, - [subject_scope, subject] - ]}, - condition - ]} + defp parse_contract({:__block__, _, ast}, contract) do + parse_ast_block(ast, contract) end - defp do_aggregate_condition(condition, subject_scope, subject) do - Macro.postwalk(condition, &to_boolean_expression(&1, subject_scope, subject)) + defp parse_contract(ast, contract) do + parse_ast(ast, contract) end - defp to_boolean_expression( - {{:., metadata, [{:__aliases__, _, [:Library]}, fun]}, _, args}, - subject_scope, - subject - ) do - arguments = - if :erlang.function_exported(Library, fun, length(args)) do - # If the number of arguments fullfill the function's arity (without subject) - args - else - [ - {:get_in, metadata, [{:scope, metadata, nil}, [subject_scope, subject]]} | args - ] - end + defp parse_ast_block([ast | rest], contract) do + case parse_ast(ast, contract) do + {:ok, contract} -> + parse_ast_block(rest, contract) - if fun |> Atom.to_string() |> String.ends_with?("?") do - {:==, metadata, - [ - true, - {{:., metadata, [{:__aliases__, [alias: Library], [:Library]}, fun]}, metadata, - arguments} - ]} - else - {:==, metadata, - [ - {:get_in, metadata, [{:scope, metadata, nil}, [subject_scope, subject]]}, - {{:., metadata, [{:__aliases__, [alias: Library], [:Library]}, fun]}, metadata, - arguments} - ]} + {:error, _} = e -> + e end end - defp to_boolean_expression(condition = {:%{}, _, _}, subject_scope, subject) do - {:==, [], - [ - {:get_in, [], [{:scope, [], nil}, [subject_scope, subject]]}, - condition - ]} - end - - # Flatten comparison operations - defp to_boolean_expression({op, _, [{:==, metadata, [{:get_in, _, _}, comp_a]}, comp_b]}, _, _) - when op in [:==, :>=, :<=] do - {op, metadata, [comp_a, comp_b]} - end - - defp to_boolean_expression(condition, _, _), do: condition - - @doc """ - - ## Examples + defp parse_ast_block([], contract), do: {:ok, contract} - iex> Interpreter.execute_actions(%Contract{ - ...> triggers: [ - ...> %Contract.Trigger{ - ...> type: :transaction, - ...> actions: {:=, [line: 2], - ...> [ - ...> {:scope, [line: 2], nil}, - ...> {:update_in, [line: 2], - ...> [ - ...> {:scope, [line: 2], nil}, - ...> ["next_transaction"], - ...> {:&, [line: 2], - ...> [ - ...> {{:., [line: 2], - ...> [ - ...> {:__aliases__, - ...> [alias: Archethic.Contracts.Interpreter.TransactionStatements], - ...> [:TransactionStatements]}, - ...> :set_type - ...> ]}, [line: 2], [{:&, [line: 2], [1]}, "transfer"]} - ...> ]} - ...> ]} - ...> ]} - ...> } - ...> ] - ...> }, :transaction) - %Transaction{type: :transfer, data: %TransactionData{}} + defp parse_ast(ast = {{:atom, "condition"}, _, _}, contract) do + case ConditionInterpreter.parse(ast) do + {:ok, condition_type, condition} -> + {:ok, Contract.add_condition(contract, condition_type, condition)} - iex> Interpreter.execute_actions(%Contract{ - ...> triggers: [ - ...> %Contract.Trigger{ - ...> type: :transaction, - ...> actions: {:__block__, [], [ - ...> { - ...> :=, - ...> [{:line, 2}], - ...> [ - ...> {:scope, [{:line, 2}], nil}, - ...> {:update_in, [line: 2], [ - ...> {:scope, [line: 2], nil}, - ...> ["next_transaction"], - ...> {:&, [line: 2], [ - ...> {{:., [line: 2], [{:__aliases__, [alias: Archethic.Contracts.Interpreter.TransactionStatements], [:TransactionStatements]}, :set_type]}, - ...> [line: 2], - ...> [{:&, [line: 2], [1]}, "transfer"]}] - ...> } - ...> ]} - ...> ] - ...> }, - ...> { - ...> :=, - ...> [line: 3], - ...> [ - ...> {:scope, [line: 3], nil}, - ...> {:update_in, [line: 3], [ - ...> {:scope, [line: 3], nil}, - ...> ["next_transaction"], - ...> {:&, [line: 3], [ - ...> { - ...> {:., [line: 3], [{:__aliases__, [alias: Archethic.Contracts.Interpreter.TransactionStatements], [:TransactionStatements]}, :add_uco_transfer]}, - ...> [line: 3], [ - ...> {:&, [line: 3], [1]}, - ...> [ - ...> {"to", "005220865F2237E3B62FFAA2AB72260A9FA711FBADF7F1DA391AB02B93D9E0D4A3"}, - ...> {"amount", 10.04} - ...> ] - ...> ] - ...> } - ...> ]} - ...> ]}] - ...> } - ...> ]}, - ...> }]}, :transaction) - %Transaction{ - type: :transfer, - data: %TransactionData{ - ledger: %Ledger{ - uco: %UCOLedger{ - transfers: [ - %UCOTransfer{ - to: <<0, 82, 32, 134, 95, 34, 55, 227, 182, 47, 250, 162, 171, 114, 38, 10, 159, 167, 17, 251, 173, 247, 241, 218, 57, 26, 176, 43, 147, 217, 224, 212, 163>>, - amount: 10.04 - } - ] - } - } - } - } - """ - def execute_actions(%Contract{triggers: triggers}, trigger_type, constants \\ %{}) do - %Contract.Trigger{actions: quoted_code} = Enum.find(triggers, &(&1.type == trigger_type)) - - {%{"next_transaction" => next_transaction}, _} = - Code.eval_quoted(quoted_code, - scope: - Map.put(constants, "next_transaction", %Transaction{ - data: %TransactionData{} - }) - ) - - next_transaction - end - - @doc """ - Execute abritary code using some constants as bindings - - ## Examples - - iex> Interpreter.execute({{:., [line: 1], - ...> [ - ...> {:__aliases__, [alias: Archethic.Contracts.Interpreter.Library], - ...> [:Library]}, - ...> :regex_match? - ...> ]}, [line: 1], - ...> [{:get_in, [line: 1], [{:scope, [line: 1], nil}, ["content"]]}, "abc"]}, %{ "content" => "abc"}) - true - - iex> Interpreter.execute({:==, [], - ...> [ - ...> {:get_in, [line: 1], [{:scope, [line: 1], nil}, ["next_transaction", "content"]]}, - ...> {:get_in, [line: 1], [{:scope, [line: 1], nil}, ["previous_transaction", "content"]]}, - ...> ]}, %{ "previous_transaction" => %{"content" => "abc"}, "next_transaction" => %{ "content" => "abc" } }) - true - - iex> Interpreter.execute({{:., [line: 2], - ...> [ - ...> {:__aliases__, [alias: Archethic.Contracts.Interpreter.Library], - ...> [:Library]}, - ...> :hash - ...> ]}, [line: 2], - ...> [{:get_in, [line: 2], [{:scope, [line: 2], nil}, ["content"]]}]}, %{ "content" => "abc" }) - <<186, 120, 22, 191, 143, 1, 207, 234, 65, 65, 64, 222, 93, 174, 34, 35, 176, 3, 97, 163, 150, 23, 122, 156, 180, 16, 255, 97, 242, 0, 21, 173>> - """ - def execute(quoted_code, constants = %{}) do - {res, _} = Code.eval_quoted(quoted_code, scope: constants) - res - end - - defp inject_bindings_and_functions(quoted_code, opts) when is_list(opts) do - bindings = Keyword.get(opts, :bindings, %{}) - subject = Keyword.get(opts, :subject) - - {ast, _} = - Macro.postwalk( - quoted_code, - %{ - bindings: bindings, - library_functions: @library_functions_names, - transaction_statements_functions: @transaction_statements_functions_names, - subject: subject - }, - &do_postwalk_execution/2 - ) - - ast - end - - defp do_postwalk_execution({:=, metadata, [var_name, content]}, acc) do - put_ast = - {{:., metadata, [{:__aliases__, metadata, [:Map]}, :put]}, metadata, - [{:scope, metadata, nil}, var_name, parse_value(content)]} - - { - {:=, metadata, [{:scope, metadata, nil}, put_ast]}, - put_in(acc, [:bindings, var_name], parse_value(content)) - } - end - - defp do_postwalk_execution(_node = {{:atom, atom}, metadata, args}, acc) - when atom in @library_functions_names do - {{{:., metadata, [{:__aliases__, [alias: Library], [:Library]}, String.to_atom(atom)]}, - metadata, args}, acc} - end - - defp do_postwalk_execution(_node = {{:atom, atom}, metadata, args}, acc) - when atom in @transaction_statements_functions_names do - args = - Enum.map(args, fn arg -> - {ast, _} = Macro.postwalk(arg, acc, &do_postwalk_execution/2) - ast - end) - - ast = { - {:., metadata, - [ - {:__aliases__, [alias: TransactionStatements], [:TransactionStatements]}, - String.to_atom(atom) - ]}, - metadata, - [{:&, metadata, [1]} | args] - } - - update_ast = - {:update_in, metadata, - [ - {:scope, metadata, nil}, - ["next_transaction"], - {:&, metadata, - [ - ast - ]} - ]} - - { - {:=, metadata, [{:scope, metadata, nil}, update_ast]}, - acc - } - end - - defp do_postwalk_execution( - _node = {{:atom, atom}, metadata, _args}, - acc = %{bindings: bindings, subject: subject} - ) do - if Map.has_key?(bindings, atom) do - search = - case subject do - nil -> - [atom] - - subject -> - # Do not use the subject when using reserved keyword - if atom in ["contract", "transaction", "next", "previous"] do - [atom] - else - [subject, atom] - end - end - - { - {:get_in, metadata, [{:scope, metadata, nil}, search]}, - acc - } - else - {atom, acc} + {:error, _} = e -> + e end end - defp do_postwalk_execution({:., metadata, [parent, {{:atom, field}}]}, acc) do - { - {:get_in, metadata, [{:scope, metadata, nil}, [parent, parse_value(field)]]}, - acc - } - end - - defp do_postwalk_execution( - {:., _, [{:get_in, metadata, [{:scope, _, nil}, access]}, {:atom, field}]}, - acc - ) do - { - {:get_in, metadata, [{:scope, metadata, nil}, access ++ [parse_value(field)]]}, - acc - } - end - - defp do_postwalk_execution( - {:., metadata, - [{:get_in, metadata, [{:scope, metadata, nil}, [parent]]}, {:atom, child}]}, - acc - ) do - {{:get_in, metadata, [{:scope, metadata, nil}, [parent, parse_value(child)]]}, acc} - end - - defp do_postwalk_execution({{:atom, atom}, val}, acc) do - {ast, _} = Macro.postwalk(val, acc, &do_postwalk_execution/2) - {{atom, ast}, acc} - end - - defp do_postwalk_execution({{:get_in, metadata, [{:scope, metadata, nil}, access]}, _, []}, acc) do - { - {:get_in, metadata, [{:scope, metadata, nil}, access]}, - acc - } - end - - defp do_postwalk_execution({:==, _, [{left, _, args_l}, {right, _, args_r}]}, acc) do - {{:==, [], [{left, [], args_l}, {right, [], args_r}]}, acc} - end + defp parse_ast(ast = {{:atom, "actions"}, _, _}, contract) do + case ActionInterpreter.parse(ast) do + {:ok, trigger_type, actions} -> + {:ok, Contract.add_trigger(contract, trigger_type, actions)} - defp do_postwalk_execution({:==, _, [{left, _, args}, right]}, acc) do - {{:==, [], [{left, [], args}, right]}, acc} - end - - defp do_postwalk_execution({node, _, _}, acc) when is_binary(node) do - {parse_value(node), acc} - end - - defp do_postwalk_execution(node, acc), do: {parse_value(node), acc} - - defp parse_value(val) when is_binary(val) do - case Base.decode16(val, case: :mixed) do - {:ok, bin} -> - bin - - _ -> - val + {:error, _} = e -> + e end end - defp parse_value(val), do: val - - @spec valid_conditions?(Conditions.t(), map()) :: boolean() - def valid_conditions?(conditions = %Conditions{}, constants = %{}) do - result = - conditions - |> Map.from_struct() - |> Enum.all?(fn {field, condition} -> - case validate_condition({field, condition}, constants) do - {_, true} -> - true - - {_, false} -> - Logger.debug( - "Invalid condition for `#{field}` with the given value: `#{get_in(constants, ["next", field])}` - expected: #{inspect(condition)}" - ) - - false - end - end) - - if result do - result - else - result - end - end - - defp validate_condition({:origin_family, _}, _) do - # Skip the verification - # The Proof of Work algorithm will use this condition to verify the transaction - {:origin_family, true} - end - - defp validate_condition({:previous_public_key, nil}, _) do - # Skip the verification as previous public key change for each transaction - {:previous_public_key, true} - end - - defp validate_condition({:timestamp, nil}, _) do - # Skip the verification as timestamp change for each transaction - {:timestamp, true} - end - - defp validate_condition({:type, nil}, %{"next" => %{"type" => "transfer"}}) do - # Skip the verification when it's the default type - {:type, true} - end - - defp validate_condition({:content, nil}, %{"next" => %{"content" => ""}}) do - # Skip the verification when it's the default type - {:content, true} - end - - # Validation rules for inherit constraints - defp validate_condition({field, nil}, %{"previous" => prev, "next" => next}) do - {field, Map.get(prev, Atom.to_string(field)) == Map.get(next, Atom.to_string(field))} - end - - defp validate_condition({field, condition}, constants = %{"next" => next}) do - result = execute(condition, constants) - - if is_boolean(result) do - {field, result} - else - {field, Map.get(next, Atom.to_string(field)) == result} - end - end - - # Validation rules for incoming transaction - defp validate_condition({field, nil}, %{"transaction" => _}) do - # Skip the validation if no transaction conditions are provided - {field, true} - end - - defp validate_condition( - {field, condition}, - constants = %{"transaction" => transaction} - ) do - result = execute(condition, constants) - - if is_boolean(result) do - {field, result} - else - {field, Map.get(transaction, Atom.to_string(field)) == result} - end - end + defp parse_ast(ast, _), do: {:error, {:unexpected_term, ast}} end diff --git a/lib/archethic/contracts/interpreter/action.ex b/lib/archethic/contracts/interpreter/action.ex new file mode 100644 index 0000000000..077c77e4b6 --- /dev/null +++ b/lib/archethic/contracts/interpreter/action.ex @@ -0,0 +1,338 @@ +defmodule Archethic.Contracts.ActionInterpreter do + @moduledoc false + + alias Archethic.Contracts.Interpreter.TransactionStatements + alias Archethic.Contracts.Interpreter.Utils, as: InterpreterUtils + + alias Archethic.TransactionChain.Transaction + alias Archethic.TransactionChain.TransactionData + + alias Crontab.CronExpression.Parser, as: CronParser + + @transaction_fields InterpreterUtils.transaction_fields() + + @transaction_statements_functions_names TransactionStatements.__info__(:functions) + |> Enum.map(&Atom.to_string(elem(&1, 0))) + + @type trigger :: :transaction | {:interval, String.t()} | {:datetime, DateTime.t()} | :oracle + + @doc ~S""" + Parse an action block and return the trigger's type associated with the code to execute + + ## Examples + + iex> ActionInterpreter.parse({{:atom, "actions"}, [line: 1], + ...> [ + ...> [ + ...> {{:atom, "triggered_by"}, {{:atom, "transaction"}, [line: 1], nil}} + ...> ], + ...> [ + ...> do: {{:atom, "add_uco_transfer"}, [line: 2], + ...> [ + ...> [ + ...> {{:atom, "to"}, "0000D574D171A484F8DEAC2D61FC3F7CC984BEB52465D69B3B5F670090742CBF5CC"}, + ...> {{:atom, "amount"}, 2000000000} + ...> ] + ...> ]} + ...> ] + ...> ]}) + {:ok, :transaction, {:=, [line: 2], [{:scope, [line: 2], nil}, {:update_in, [line: 2], [{:scope, [line: 2], nil}, ["next_transaction"], {:&, [line: 2], [{{:., [line: 2], [{:__aliases__, [alias: Archethic.Contracts.Interpreter.TransactionStatements], [:TransactionStatements]}, :add_uco_transfer]}, [line: 2], [{:&, [line: 2], [1]}, [{"to", "0000D574D171A484F8DEAC2D61FC3F7CC984BEB52465D69B3B5F670090742CBF5CC"}, {"amount", 2000000000}]]}]}]}]}} + + Usage with trigger accepting parameters + + iex> ActionInterpreter.parse({{:atom, "actions"}, [line: 1], + ...> [ + ...> [ + ...> {{:atom, "triggered_by"}, {{:atom, "datetime"}, + ...> [line: 1], nil}}, + ...> {{:atom, "at"}, 1391309030} + ...> ], + ...> [ + ...> do: {{:atom, "add_recipient"}, [line: 2], + ...> ["0000D574D171A484F8DEAC2D61FC3F7CC984BEB52465D69B3B5F670090742CBF5CC"]} + ...> ] + ...> ]}) + {:ok, {:datetime, ~U[2014-02-02 02:43:50Z]}, {:=, [line: 2], [{:scope, [line: 2], nil}, {:update_in, [line: 2], [{:scope, [line: 2], nil}, ["next_transaction"], {:&, [line: 2], [{{:., [line: 2], [{:__aliases__, [alias: Archethic.Contracts.Interpreter.TransactionStatements], [:TransactionStatements]}, :add_recipient]}, [line: 2], [{:&, [line: 2], [1]}, "0000D574D171A484F8DEAC2D61FC3F7CC984BEB52465D69B3B5F670090742CBF5CC"]}]}]}]}} + + + Prevent usage of not authorized functions + + iex> ActionInterpreter.parse({{:atom, "actions"}, [line: 1], + ...> [ + ...> [{{:atom, "triggered_by"}, {{:atom, "transaction"}, [line: 1], nil}}], + ...> [ + ...> do: {{:., [line: 2], + ...> [{:__aliases__, [line: 2], [atom: "System"]}, {:atom, "user_home"}]}, + ...> [line: 2], []} + ...> ] + ...> ]} + ...> ) + {:error, "unexpected term - System - L2"} + + """ + @spec parse(any()) :: {:ok, trigger(), Macro.t()} | {:error, String.t()} + def parse(ast) do + case Macro.traverse( + ast, + {:ok, %{scope: :root}}, + &prewalk(&1, &2), + &postwalk/2 + ) do + {_node, {:ok, trigger, actions}} -> + {:ok, trigger, actions} + + {node, _} -> + {:error, InterpreterUtils.format_error_reason(node, "unexpected term")} + end + catch + {:error, reason, node} -> + {:error, InterpreterUtils.format_error_reason(node, reason)} + + {:error, node} -> + {:error, InterpreterUtils.format_error_reason(node, "unexpected term")} + end + + # Whitelist the actions DSL + defp prewalk(node = {{:atom, "actions"}, _, _}, {:ok, context = %{scope: :root}}) do + {node, {:ok, %{context | scope: :actions}}} + end + + # Whitelist the triggers + defp prewalk( + node = {{:atom, "triggered_by"}, {{:atom, trigger}, _, _}}, + {:ok, context = %{scope: :actions}} + ) + when trigger in ["transaction", "datetime", "interval", "oracle"] do + {node, {:ok, %{context | scope: {:actions, String.to_existing_atom(trigger)}}}} + end + + defp prewalk(node = {{:atom, "at"}, timestamp}, acc = {:ok, %{scope: {:actions, :datetime}}}) do + with digits when length(digits) == 10 <- Integer.digits(timestamp), + {:ok, _} <- DateTime.from_unix(timestamp) do + {node, acc} + else + _ -> + {node, {:error, "invalid datetime's trigger"}} + end + end + + defp prewalk(node = {{:atom, "at"}, interval}, acc = {:ok, %{scope: {:actions, :interval}}}) do + case CronParser.parse(interval) do + {:ok, _} -> + {node, acc} + + {:error, _} -> + {node, {:error, "invalid interval"}} + end + end + + # Whitelist variable assignation inside the actions + defp prewalk(node = {:=, _, _}, acc = {:ok, %{scope: {:actions, _}}}), do: {node, acc} + + # Whitelist the transaction statements functions + defp prewalk( + node = {{:atom, function}, _, _}, + {:ok, context = %{scope: parent_scope = {:actions, _}}} + ) + when function in @transaction_statements_functions_names do + {node, {:ok, %{context | scope: {:function, function, parent_scope}}}} + end + + # Whitelist the add_uco_transfer function parameters + defp prewalk( + node = {{:atom, "to"}, address}, + acc = {:ok, %{scope: {"add_uco_transfer", {:actions, _}}}} + ) + when is_binary(address) do + {node, acc} + end + + defp prewalk( + node = {{:atom, "to"}, address}, + acc = {:ok, %{scope: {:function, "add_uco_transfer", {:actions, _}}}} + ) + when is_binary(address) do + {node, acc} + end + + defp prewalk( + node = {{:atom, "to"}, {{:atom, _}, _, _}}, + acc = {:ok, %{scope: {:function, "add_uco_transfer", {:actions, _}}}} + ) do + {node, acc} + end + + defp prewalk( + node = {{:atom, "amount"}, amount}, + acc = {:ok, %{scope: {:function, "add_uco_transfer", {:actions, _}}}} + ) + when is_integer(amount) and amount > 0 do + {node, acc} + end + + defp prewalk( + node = {{:atom, "amount"}, {{:atom, _}, _, _}}, + acc = {:ok, %{scope: {:function, "add_uco_transfer", {:actions, _}}}} + ) do + {node, acc} + end + + # Whitelist the add_token_transfer argument list + defp prewalk( + node = {{:atom, "to"}, address}, + acc = {:ok, %{scope: {:function, "add_token_transfer", {:actions, _}}}} + ) + when is_binary(address) do + {node, acc} + end + + defp prewalk( + node = {{:atom, "to"}, {{:atom, _}, _, _}}, + acc = {:ok, %{scope: {:function, "add_token_transfer", {:actions, _}}}} + ) do + {node, acc} + end + + defp prewalk( + node = {{:atom, "amount"}, amount}, + acc = {:ok, %{scope: {:function, "add_token_transfer", {:actions, _}}}} + ) + when is_integer(amount) and amount > 0 do + {node, acc} + end + + defp prewalk( + node = {{:atom, "amount"}, {{:atom, _}, _, _}}, + acc = {:ok, %{scope: {:function, "add_token_transfer", {:actions, _}}}} + ) do + {node, acc} + end + + defp prewalk( + node = {{:atom, "token_address"}, token_address}, + acc = {:ok, %{scope: {:function, "add_token_transfer", {:actions, _}}}} + ) + when is_binary(token_address) do + {node, acc} + end + + defp prewalk( + node = {{:atom, "token_address"}, {{:atom, _}, _, _}}, + acc = {:ok, %{scope: {:function, "add_token_transfer", {:actions, _}}}} + ) do + {node, acc} + end + + defp prewalk( + node = {{:atom, "token_id"}, token_id}, + acc = {:ok, %{scope: {:function, "add_token_transfer", {:actions, _}}}} + ) + when is_integer(token_id) and token_id >= 0 do + {node, acc} + end + + defp prewalk( + node = {{:atom, "token_id"}, {{:atom, _}, _, _}}, + acc = {:ok, %{scope: {:function, "add_token_transfer", {:actions, _}}}} + ) do + {node, acc} + end + + # Whitelist the add_ownership argument list + defp prewalk( + node = {{:atom, "secret"}, secret}, + acc = {:ok, %{scope: {:function, "add_ownership", {:actions, _}}}} + ) + when is_binary(secret) do + {node, acc} + end + + defp prewalk( + node = {{:atom, "secret"}, {{:atom, _}, _, _}}, + acc = {:ok, %{scope: {:function, "add_ownership", {:actions, _}}}} + ) do + {node, acc} + end + + defp prewalk( + node = {{:atom, "secret_key"}, _secret_key}, + acc = {:ok, %{scope: {:function, "add_ownership", {:actions, _}}}} + ) do + {node, acc} + end + + defp prewalk( + node = {{:atom, "authorized_public_keys"}, authorized_public_keys}, + acc = {:ok, %{scope: {:function, "add_ownership", {:actions, _}}}} + ) + when is_list(authorized_public_keys) do + {node, acc} + end + + defp prewalk( + node = {{:atom, "authorized_public_keys"}, {{:atom, _, _}}}, + acc = {:ok, %{scope: {:function, "add_ownership", {:actions, _}}}} + ) do + {node, acc} + end + + defp prewalk(node, {:error, reason}) do + throw({:error, reason, node}) + end + + defp prewalk(node, acc) do + InterpreterUtils.prewalk(node, acc) + end + + defp postwalk( + node = + {{:atom, "actions"}, [line: _], + [[{{:atom, "triggered_by"}, {{:atom, trigger_type}, _, _}} | opts], [do: actions]]}, + {:ok, _} + ) do + actions = + InterpreterUtils.inject_bindings_and_functions(actions, + bindings: %{ + "contract" => Enum.map(@transaction_fields, &{&1, ""}) |> Enum.into(%{}), + "transaction" => Enum.map(@transaction_fields, &{&1, ""}) |> Enum.into(%{}) + } + ) + + case trigger_type do + "transaction" -> + {node, {:ok, :transaction, actions}} + + "datetime" -> + [{{:atom, "at"}, timestamp}] = opts + datetime = DateTime.from_unix!(timestamp) + {node, {:ok, {:datetime, datetime}, actions}} + + "interval" -> + [{{:atom, "at"}, interval}] = opts + {node, {:ok, {:interval, interval}, actions}} + + "oracle" -> + {node, {:ok, :oracle, actions}} + end + end + + defp postwalk(node, acc) do + InterpreterUtils.postwalk(node, acc) + end + + @doc """ + Execute actions code and returns a transaction as result + """ + @spec execute(Macro.t(), map()) :: Transaction.t() + def execute(code, constants \\ %{}) do + {%{"next_transaction" => next_transaction}, _} = + Code.eval_quoted(code, + scope: + Map.put(constants, "next_transaction", %Transaction{ + data: %TransactionData{} + }) + ) + + next_transaction + end +end diff --git a/lib/archethic/contracts/interpreter/condition.ex b/lib/archethic/contracts/interpreter/condition.ex new file mode 100644 index 0000000000..14c887c45b --- /dev/null +++ b/lib/archethic/contracts/interpreter/condition.ex @@ -0,0 +1,601 @@ +defmodule Archethic.Contracts.ConditionInterpreter do + @moduledoc false + + alias Archethic.Contracts.ContractConditions, as: Conditions + alias Archethic.Contracts.ContractConstants, as: Constants + alias Archethic.Contracts.Interpreter.Library + alias Archethic.Contracts.Interpreter.Utils, as: InterpreterUtils + alias Archethic.SharedSecrets + + @condition_fields Conditions.__struct__() + |> Map.keys() + |> Enum.reject(&(&1 == :__struct__)) + |> Enum.map(&Atom.to_string/1) + + @transaction_fields InterpreterUtils.transaction_fields() + + @exported_library_functions Library.__info__(:functions) + + @type condition_type :: :transaction | :inherit | :oracle + + require Logger + + @doc ~S""" + Parse a condition block and returns the right condition's type with a `Archethic.Contracts.Contract.Conditions` struct + + ## Examples + + iex> ConditionInterpreter.parse({{:atom, "condition"}, [line: 1], + ...> [ + ...> [ + ...> {{:atom, "transaction"}, [ + ...> {{:atom, "content"}, "hello"} + ...> ]} + ...> ] + ...> ]}) + {:ok, :transaction, %Conditions{ + content: {:==, [], [ + {:get_in, [], [ + {:scope, [], nil}, + ["transaction", "content"] + ]}, + "hello" + ]} + } + } + + Usage of functions in the condition fields + + iex> ConditionInterpreter.parse({{:atom, "condition"}, [line: 1], + ...> [ + ...> [ + ...> {{:atom, "transaction"}, [ + ...> {{:atom, "content"}, {{:atom, "hash"}, [line: 2], + ...> [ + ...> {{:., [line: 2], + ...> [ + ...> { {:atom, "contract"}, [line: 2], + ...> nil}, + ...> {:atom, "code"} + ...> ]}, + ...> [no_parens: true, line: 2], + ...> []} + ...> ]} + ...> }]} + ...> ] + ...> ]}) + { + :ok, :transaction, %Conditions{ + content: {:==, [line: 2], [ + {:get_in, [line: 2], [ + {:scope, [line: 2], nil}, + ["transaction", "content"] + ]}, + { + {:., [line: 2], [ + {:__aliases__, [alias: Archethic.Contracts.Interpreter.Library], [:Library]}, + :hash + ]}, [line: 2], [ + {:get_in, [line: 2], [ + {:scope, [line: 2], nil}, + ["contract", "code"] + ]} + ] + } + ] + } + } + } + + Usage with multiple condition fields + + iex> ConditionInterpreter.parse({{:atom, "condition"}, [line: 1], + ...> [ + ...> {{:atom, "transaction"}, [ + ...> {{:atom, "content"}, "hello"}, + ...> {{:atom, "uco_transfers"}, {:%{}, [line: 3], + ...> [ + ...> {"00006B368BE45DACD0CBC0EC5893BDC1079448181AA88A2CBB84AF939912E858843E", + ...> 1000000000} + ...> ]} + ...> } + ...> ]} + ...> ]}) + {:ok, :transaction, %Conditions{ + content: {:==, [], [{:get_in, [], [{:scope, [], nil}, ["transaction", "content"]]}, "hello"]}, + uco_transfers: {:==, [], [ + {:get_in, [], [{:scope, [], nil}, + ["transaction", "uco_transfers"] + ]}, + {:%{}, [line: 3], [{ + "00006B368BE45DACD0CBC0EC5893BDC1079448181AA88A2CBB84AF939912E858843E", 1000000000 + }]} + ]} + } + } + + Usage with origin_family condition + + iex> ConditionInterpreter.parse({{:atom, "condition"}, [line: 1], + ...> [ + ...> [ + ...> {{:atom, "inherit"}, [ + ...> {{:atom, "origin_family"}, {{:atom, "abc"}, + ...> [line: 2], nil}} + ...> ]} + ...> ] + ...> ]}) + {:error, "invalid origin family - L2"} + + """ + @spec parse(any()) :: + {:ok, condition_type(), Conditions.t()} | {:error, reason :: String.t()} + def parse(ast) do + case Macro.traverse( + ast, + {:ok, %{scope: :root}}, + &prewalk(&1, &2), + &postwalk/2 + ) do + {_node, {:ok, condition_name, conditions}} -> + {:ok, condition_name, conditions} + + {node, _} -> + {:error, InterpreterUtils.format_error_reason(node, "unexpected term")} + end + catch + {:error, node} -> + {:error, InterpreterUtils.format_error_reason(node, "unexpected term")} + + {:error, reason, node} -> + {:error, InterpreterUtils.format_error_reason(node, reason)} + end + + # Whitelist the DSL for conditions + defp prewalk( + node = {{:atom, "condition"}, _metadata, _}, + {:ok, context = %{scope: :root}} + ) do + {node, {:ok, %{context | scope: :condition}}} + end + + # Whitelist the transaction/inherit/oracle conditions + defp prewalk(node = {{:atom, condition_name}, rest}, {:ok, context = %{scope: :condition}}) + when condition_name in ["transaction", "inherit", "oracle"] and is_list(rest), + do: + {node, {:ok, %{context | scope: {:condition, String.to_existing_atom(condition_name)}}}} + + # Whitelist the transaction fields in the conditions + defp prewalk( + node = {{:atom, field}, _}, + {:ok, context = %{scope: {:condition, condition_name}}} + ) + when field in @condition_fields do + {node, {:ok, %{context | scope: {:condition, condition_name, field}}}} + end + + # Whitelist the origin family + defp prewalk(node = [{{:atom, "origin_family"}, {{:atom, family}, _, _}}], acc = {:ok, _}) do + families = SharedSecrets.list_origin_families() |> Enum.map(&Atom.to_string/1) + + if family in families do + {node, acc} + else + {node, {:error, "invalid origin family"}} + end + end + + defp prewalk(node = [{{:atom, "uco_transfers"}, value}], acc = {:ok, _}) do + case value do + {:%{}, _, _} -> + {node, acc} + + {op, _, _} when op in [:==, :<, :>, :<=, :>=, :if] -> + {node, acc} + + _ -> + {node, {:error, "must be a map or a code instruction starting by an comparator"}} + end + end + + defp prewalk(node = [{{:atom, "token_transfers"}, value}], acc = {:ok, _}) do + case value do + {:%{}, _, _} -> + {node, acc} + + {op, _, _} when op in [:==, :<, :>, :<=, :>=, :if] -> + {node, acc} + + _ -> + {node, {:error, "must be a map or a code instruction starting by an comparator"}} + end + end + + # Whitelist the regex_match?/1 function in the condition + defp prewalk( + node = {{:atom, "regex_match?"}, _, [_search]}, + acc = {:ok, %{scope: {:condition, _, _}}} + ) do + {node, acc} + end + + # Whitelist the json_path_extract/1 function in the condition + defp prewalk( + node = {{:atom, "json_path_extract"}, _, [_search]}, + acc = {:ok, %{scope: {:condition, _, _}}} + ) do + {node, acc} + end + + # Whitelist the json_path_match?/1 function in the condition + defp prewalk( + node = {{:atom, "json_path_match?"}, _, [_search]}, + acc = {:ok, %{scope: {:condition, _, _}}} + ) do + {node, acc} + end + + # Whitelist the hash/0 function in the condition + defp prewalk( + node = {{:atom, "hash"}, _, []}, + acc = {:ok, %{scope: {:condition, _, _}}} + ) do + {node, acc} + end + + # Whitelist the in?/1 function in the condition + defp prewalk( + node = {{:atom, "in?"}, _, [_data]}, + acc = {:ok, %{scope: {:condition, _, _}}} + ) do + {node, acc} + end + + # Whitelist the size/0 function in the condition + defp prewalk(node = {{:atom, "size"}, _, []}, acc = {:ok, %{scope: {:condition, _, _}}}), + do: {node, acc} + + # Whitelist the get_genesis_address/0 function in condition + defp prewalk( + node = {{:atom, "get_genesis_address"}, _, []}, + acc = {:ok, %{scope: {:condition, _, _}}} + ) do + {node, acc} + end + + # Whitelist the get_genesis_public_key/0 function in condition + defp prewalk( + node = {{:atom, "get_genesis_public_key"}, _, []}, + acc = {:ok, %{scope: {:condition, _, _}}} + ) do + {node, acc} + end + + # Whitelist usage of taps in the field of a condition + defp prewalk(node = {{:atom, _key}, _val}, acc = {:ok, %{scope: {:condition, _, _}}}) do + {node, acc} + end + + defp prewalk(node, {:error, reason}) do + throw({:error, reason, node}) + end + + defp prewalk(node, acc) do + InterpreterUtils.prewalk(node, acc) + end + + defp postwalk(node, :error), do: {node, :error} + + defp postwalk( + node = {{:atom, "condition"}, _, [[{{:atom, condition_name}, conditions}]]}, + {:ok, _} + ) do + conditions = build_conditions(condition_name, conditions) + + acc = + case condition_name do + "transaction" -> {:ok, :transaction, conditions} + "inherit" -> {:ok, :inherit, conditions} + "oracle" -> {:ok, :oracle, conditions} + end + + {node, acc} + end + + defp postwalk( + node = {{:atom, "condition"}, _, [{{:atom, condition_name}, conditions}]}, + {:ok, _} + ) do + conditions = build_conditions(condition_name, conditions) + + acc = + case condition_name do + "transaction" -> {:ok, :transaction, conditions} + "inherit" -> {:ok, :inherit, conditions} + "oracle" -> {:ok, :oracle, conditions} + end + + {node, acc} + end + + defp postwalk( + node = + {{:atom, "condition"}, _, + [ + {{:atom, condition_name}, _, + [ + conditions + ]} + ]}, + {:ok, _} + ) do + conditions = build_conditions(condition_name, conditions) + + acc = + case condition_name do + "transaction" -> {:ok, :transaction, conditions} + "inherit" -> {:ok, :inherit, conditions} + "oracle" -> {:ok, :oracle, conditions} + end + + {node, acc} + end + + defp postwalk( + node = {{:atom, field}, _}, + {:ok, context = %{scope: {:condition, condition_name, field}}} + ) + when field in @condition_fields do + {node, {:ok, %{context | scope: {:condition, condition_name}}}} + end + + defp postwalk( + node = {{:atom, condition_name}, _}, + {:ok, context = %{scope: {:condition, _}}} + ) + when condition_name in ["transaction", "inherit", "oracle"] do + {node, {:ok, %{context | scope: :condition}}} + end + + defp postwalk(node, acc) do + InterpreterUtils.postwalk(node, acc) + end + + defp build_conditions(condition_name, conditions) do + bindings = Enum.map(@transaction_fields, &{&1, ""}) |> Enum.into(%{}) + + bindings = + case condition_name do + "inherit" -> + Map.merge(bindings, %{ + "next" => Enum.map(@transaction_fields, &{&1, ""}) |> Enum.into(%{}), + "previous" => Enum.map(@transaction_fields, &{&1, ""}) |> Enum.into(%{}) + }) + + _ -> + Map.merge(bindings, %{ + "contract" => Enum.map(@transaction_fields, &{&1, ""}) |> Enum.into(%{}), + "transaction" => Enum.map(@transaction_fields, &{&1, ""}) |> Enum.into(%{}) + }) + end + + subject_scope = if condition_name == "inherit", do: "next", else: "transaction" + + conditions + |> InterpreterUtils.inject_bindings_and_functions( + bindings: bindings, + subject: subject_scope + ) + |> aggregate_conditions(subject_scope) + end + + defp aggregate_conditions(conditions, subject_scope) do + Enum.reduce(conditions, %Conditions{}, fn {subject, condition}, acc -> + condition = do_aggregate_condition(condition, subject_scope, subject) + Map.put(acc, String.to_existing_atom(subject), condition) + end) + end + + defp do_aggregate_condition(condition, _, "origin_family"), + do: String.to_existing_atom(condition) + + defp do_aggregate_condition(condition, subject_scope, subject) + when is_binary(condition) or is_number(condition) do + {:==, [], + [ + {:get_in, [], [{:scope, [], nil}, [subject_scope, subject]]}, + condition + ]} + end + + defp do_aggregate_condition(condition, subject_scope, subject) when is_list(condition) do + {:==, [], + [ + {:get_in, [], + [ + {:scope, [], nil}, + [subject_scope, subject] + ]}, + condition + ]} + end + + defp do_aggregate_condition(condition, subject_scope, subject) do + Macro.postwalk(condition, &to_boolean_expression(&1, subject_scope, subject)) + end + + defp to_boolean_expression( + {{:., metadata, [{:__aliases__, _, [:Library]}, fun]}, _, args}, + subject_scope, + subject + ) do + arg_length = length(args) + + arguments = + case Keyword.get(@exported_library_functions, fun) do + ^arg_length -> + # If the number of arguments fullfill the function's arity (without subject) + args + + _ -> + [ + {:get_in, metadata, [{:scope, metadata, nil}, [subject_scope, subject]]} | args + ] + end + + if fun |> Atom.to_string() |> String.ends_with?("?") do + {:==, metadata, + [ + true, + {{:., metadata, [{:__aliases__, [alias: Library], [:Library]}, fun]}, metadata, + arguments} + ]} + else + {:==, metadata, + [ + {:get_in, metadata, [{:scope, metadata, nil}, [subject_scope, subject]]}, + {{:., metadata, [{:__aliases__, [alias: Library], [:Library]}, fun]}, metadata, + arguments} + ]} + end + end + + defp to_boolean_expression(condition = {:%{}, _, _}, subject_scope, subject) do + {:==, [], + [ + {:get_in, [], [{:scope, [], nil}, [subject_scope, subject]]}, + condition + ]} + end + + # Flatten comparison operations + defp to_boolean_expression({op, _, [{:==, metadata, [{:get_in, _, _}, comp_a]}, comp_b]}, _, _) + when op in [:==, :>=, :<=] do + {op, metadata, [comp_a, comp_b]} + end + + defp to_boolean_expression(condition, _, _), do: condition + + @doc """ + Determines if the conditions of a contract are valid from the given constants + """ + @spec valid_conditions?(Conditions.t(), map()) :: boolean() + def valid_conditions?(conditions = %Conditions{}, constants = %{}) do + constants = + constants + |> Enum.map(fn {subset, constants} -> + {subset, Constants.stringify(constants)} + end) + |> Enum.into(%{}) + + result = + conditions + |> Map.from_struct() + |> Enum.all?(fn {field, condition} -> + field = Atom.to_string(field) + + case validate_condition({field, condition}, constants) do + {_, true} -> + true + + {_, false} -> + value = get_constant_value(constants, field) + + Logger.debug( + "Invalid condition for `#{inspect(field)}` with the given value: `#{inspect(value)}` - condition: #{inspect(condition)}" + ) + + false + end + end) + + if result do + result + else + result + end + end + + defp get_constant_value(constants, field) do + case get_in(constants, [ + Access.key("transaction", %{}), + Access.key(field, "") + ]) do + "" -> + get_in(constants, ["next", field]) + + value -> + value + end + end + + defp validate_condition({"origin_family", _}, _) do + # Skip the verification + # The Proof of Work algorithm will use this condition to verify the transaction + {"origin_family", true} + end + + defp validate_condition({"address", nil}, _) do + # Skip the verification as the address changes for each transaction + {"address", true} + end + + defp validate_condition({"previous_public_key", nil}, _) do + # Skip the verification as the previous public key changes for each transaction + {"previous_public_key", true} + end + + defp validate_condition({"timestamp", nil}, _) do + # Skip the verification as timestamp changes for each transaction + {"timestamp", true} + end + + defp validate_condition({"type", nil}, %{"next" => %{"type" => "transfer"}}) do + # Skip the verification when it's the default type + {"type", true} + end + + defp validate_condition({"content", nil}, %{"next" => %{"content" => ""}}) do + # Skip the verification when it's the default type + {"content", true} + end + + # Validation rules for inherit constraints + defp validate_condition({field, nil}, %{"previous" => prev, "next" => next}) do + {field, Map.get(prev, field) == Map.get(next, field)} + end + + defp validate_condition({field, condition}, constants = %{"next" => next}) do + result = execute_condition_code(condition, constants) + + if is_boolean(result) do + {field, result} + else + {field, Map.get(next, field) == result} + end + end + + # Validation rules for incoming transaction + defp validate_condition({field, nil}, %{"transaction" => _}) do + # Skip the validation if no transaction conditions are provided + {field, true} + end + + defp validate_condition( + {field, condition}, + constants = %{"transaction" => transaction} + ) do + result = execute_condition_code(condition, constants) + + if is_boolean(result) do + {field, result} + else + {field, Map.get(transaction, field) == result} + end + end + + defp execute_condition_code(quoted_code, constants) do + {res, _} = Code.eval_quoted(quoted_code, scope: constants) + res + end +end diff --git a/lib/archethic/contracts/interpreter/library.ex b/lib/archethic/contracts/interpreter/library.ex index eee53fc404..8256439d7a 100644 --- a/lib/archethic/contracts/interpreter/library.ex +++ b/lib/archethic/contracts/interpreter/library.ex @@ -109,12 +109,11 @@ defmodule Archethic.Contracts.Interpreter.Library do ## Examples iex> Library.hash("hello") - <<44, 242, 77, 186, 95, 176, 163, 14, 38, 232, 59, 42, 197, 185, 226, 158, - 27, 22, 30, 92, 31, 167, 66, 94, 115, 4, 51, 98, 147, 139, 152, 36>> + "2CF24DBA5FB0A30E26E83B2AC5B9E29E1B161E5C1FA7425E73043362938B9824" """ @spec hash(binary()) :: binary() def hash(content) when is_binary(content) do - :crypto.hash(:sha256, content) + :crypto.hash(:sha256, decode_binary(content)) |> Base.encode16() end @doc """ @@ -152,7 +151,7 @@ defmodule Archethic.Contracts.Interpreter.Library do 2 """ @spec size(binary() | list()) :: non_neg_integer() - def size(binary) when is_binary(binary), do: byte_size(binary) + def size(binary) when is_binary(binary), do: binary |> decode_binary() |> byte_size() def size(list) when is_list(list), do: length(list) def size(map) when is_map(map), do: map_size(map) @@ -163,9 +162,10 @@ defmodule Archethic.Contracts.Interpreter.Library do @spec get_genesis_address(binary()) :: binary() def get_genesis_address(address) do - nodes = Election.chain_storage_nodes(address, P2P.authorized_and_available_nodes()) - {:ok, address} = download_first_address(nodes, address) - address + bin_address = decode_binary(address) + nodes = Election.chain_storage_nodes(bin_address, P2P.authorized_and_available_nodes()) + {:ok, address} = download_first_address(nodes, bin_address) + Base.encode16(address) end @doc """ @@ -173,9 +173,10 @@ defmodule Archethic.Contracts.Interpreter.Library do """ @spec get_genesis_public_key(binary()) :: binary() def get_genesis_public_key(address) do - nodes = Election.chain_storage_nodes(address, P2P.authorized_and_available_nodes()) - {:ok, key} = download_first_public_key(nodes, address) - key + bin_address = decode_binary(address) + nodes = Election.chain_storage_nodes(bin_address, P2P.authorized_and_available_nodes()) + {:ok, key} = download_first_public_key(nodes, bin_address) + Base.encode16(key) end defp download_first_public_key([node | rest], public_key) do @@ -202,4 +203,18 @@ defmodule Archethic.Contracts.Interpreter.Library do """ @spec timestamp() :: non_neg_integer() def timestamp, do: DateTime.utc_now() |> DateTime.to_unix() + + defp decode_binary(bin) do + if String.printable?(bin) do + case Base.decode16(bin, case: :mixed) do + {:ok, hex} -> + hex + + _ -> + bin + end + else + bin + end + end end diff --git a/lib/archethic/contracts/interpreter/transaction_statements.ex b/lib/archethic/contracts/interpreter/transaction_statements.ex index 94c08a12d2..28f114b1a0 100644 --- a/lib/archethic/contracts/interpreter/transaction_statements.ex +++ b/lib/archethic/contracts/interpreter/transaction_statements.ex @@ -165,9 +165,9 @@ defmodule Archethic.Contracts.Interpreter.TransactionStatements do ownership = Ownership.new( - secret, - secret_key, - Enum.map(authorized_public_keys, &Base.decode16!(&1, case: :mixed)) + decode_binary(secret), + decode_binary(secret_key), + Enum.map(authorized_public_keys, &decode_binary(&1)) ) update_in( @@ -201,14 +201,8 @@ defmodule Archethic.Contracts.Interpreter.TransactionStatements do end defp decode_binary(bin) do - if String.printable?(bin) do - case Base.decode16(bin, case: :mixed) do - {:ok, hex} -> - hex - - _ -> - bin - end + if String.match?(bin, ~r/^[[:xdigit:]]+$/) do + Base.decode16!(bin, case: :mixed) else bin end diff --git a/lib/archethic/contracts/interpreter/utils.ex b/lib/archethic/contracts/interpreter/utils.ex new file mode 100644 index 0000000000..93e24599ea --- /dev/null +++ b/lib/archethic/contracts/interpreter/utils.ex @@ -0,0 +1,547 @@ +defmodule Archethic.Contracts.Interpreter.Utils do + @moduledoc false + + alias Archethic.Contracts.Interpreter.Library + alias Archethic.Contracts.Interpreter.TransactionStatements + + @library_functions_names Library.__info__(:functions) + |> Enum.map(&Atom.to_string(elem(&1, 0))) + + @library_functions_names_atoms Library.__info__(:functions) + |> Enum.map(&{Atom.to_string(elem(&1, 0)), elem(&1, 0)}) + |> Enum.into(%{}) + + @transaction_statements_functions_names TransactionStatements.__info__(:functions) + |> Enum.map(&Atom.to_string(elem(&1, 0))) + + @transaction_statements_functions_names_atoms TransactionStatements.__info__(:functions) + |> Enum.map( + &{Atom.to_string(elem(&1, 0)), elem(&1, 0)} + ) + |> Enum.into(%{}) + + @transaction_fields [ + "address", + "type", + "timestamp", + "previous_signature", + "previous_public_key", + "origin_signature", + "content", + "keys", + "code", + "uco_ledger", + "token_ledger", + "uco_transfers", + "token_transfers", + "authorized_public_keys", + "secrets", + "recipients" + ] + + @spec transaction_fields() :: list(String.t()) + def transaction_fields, do: @transaction_fields + + @spec prewalk(Macro.t(), any()) :: {Macro.t(), any()} + def prewalk(node = :atom, acc), do: {node, acc} + def prewalk(node = {:atom, key}, acc) when is_binary(key), do: {node, acc} + + def prewalk(node = {{:atom, key}, _, nil}, acc = {:ok, _}) when is_binary(key), + do: {node, acc} + + def prewalk(node, acc) when is_list(node), do: {node, acc} + + # Whitelist operators + def prewalk(node = {:+, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, + do: {node, acc} + + def prewalk(node = {:-, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, + do: {node, acc} + + def prewalk(node = {:/, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, + do: {node, acc} + + def prewalk(node = {:*, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, + do: {node, acc} + + def prewalk(node = {:>, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, + do: {node, acc} + + def prewalk(node = {:<, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, + do: {node, acc} + + def prewalk(node = {:>=, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, + do: {node, acc} + + def prewalk(node = {:<=, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, + do: {node, acc} + + def prewalk(node = {:|>, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, + do: {node, acc} + + def prewalk(node = {:==, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, + do: {node, acc} + + # Whitelist the use of doted statement + def prewalk(node = {{:., _, [{_, _, _}, _]}, _, []}, acc = {:ok, %{scope: scope}}) + when scope != :root, + do: {node, acc} + + def prewalk(node = {:if, _, _}, acc = {:ok, %{scope: scope}}) when scope != :root, + do: {node, acc} + + def prewalk(node = {:else, _}, acc = {:ok, %{scope: scope}}) when scope != :root, + do: {node, acc} + + def prewalk(node = [do: _, else: _], acc = {:ok, %{scope: scope}}) when scope != :root, + do: {node, acc} + + def prewalk(node = :else, acc = {:ok, %{scope: scope}}) when scope != :root, do: {node, acc} + + def prewalk(node = {:and, _, _}, acc = {:ok, _}), do: {node, acc} + def prewalk(node = {:or, _, _}, acc = {:ok, _}), do: {node, acc} + + # Whitelist the in operation + def prewalk(node = {:in, _, [_, _]}, acc = {:ok, _}), do: {node, acc} + + # Whitelist maps + def prewalk(node = {:%{}, _, fields}, acc = {:ok, _}) when is_list(fields), do: {node, acc} + + def prewalk(node = {key, _val}, acc) when is_binary(key) do + {node, acc} + end + + # Whitelist the multiline + def prewalk(node = {{:__block__, _, _}}, acc = {:ok, _}) do + {node, acc} + end + + def prewalk(node = {:__block__, _, _}, acc = {:ok, _}) do + {node, acc} + end + + # Whitelist interpolation of strings + def prewalk( + node = + {:<<>>, _, [{:"::", _, [{{:., _, [Kernel, :to_string]}, _, _}, {:binary, _, nil}]}, _]}, + acc + ) do + {node, acc} + end + + def prewalk( + node = + {:<<>>, _, + [ + _, + {:"::", _, [{{:., _, [Kernel, :to_string]}, _, _}, _]} + ]}, + acc + ) do + {node, acc} + end + + def prewalk(node = {:"::", _, [{{:., _, [Kernel, :to_string]}, _, _}, _]}, acc) do + {node, acc} + end + + def prewalk(node = {{:., _, [Kernel, :to_string]}, _, _}, acc) do + {node, acc} + end + + def prewalk(node = {:., _, [Kernel, :to_string]}, acc) do + {node, acc} + end + + def prewalk(node = Kernel, acc), do: {node, acc} + def prewalk(node = :to_string, acc), do: {node, acc} + def prewalk(node = {:binary, _, nil}, acc), do: {node, acc} + + # Whitelist generics + def prewalk(true, acc = {:ok, _}), do: {true, acc} + def prewalk(false, acc = {:ok, _}), do: {false, acc} + def prewalk(number, acc = {:ok, _}) when is_number(number), do: {number, acc} + def prewalk(string, acc = {:ok, _}) when is_binary(string), do: {string, acc} + def prewalk(node = [do: _], acc = {:ok, _}), do: {node, acc} + def prewalk(node = {:do, _}, acc = {:ok, _}), do: {node, acc} + def prewalk(node = :do, acc = {:ok, _}), do: {node, acc} + + # Whitelist the use of list + def prewalk(node = [{{:atom, _}, _, nil} | _], acc = {:ok, %{scope: scope}}) + when scope != :root do + {node, acc} + end + + # Whitelist access to map field + def prewalk( + node = {{:., _, [Access, :get]}, _, _}, + acc = {:ok, %{scope: scope}} + ) + when scope != :root do + {node, acc} + end + + def prewalk(node = {:., _, [Access, :get]}, acc = {:ok, %{scope: scope}}) when scope != :root, + do: {node, acc} + + def prewalk(node = Access, acc), do: {node, acc} + def prewalk(node = :get, acc), do: {node, acc} + + # Whitelist the usage of transaction fields in references: "transaction/contract/previous/next" + def prewalk( + node = {:., _, [{{:atom, transaction_ref}, _, nil}, {:atom, transaction_field}]}, + acc = {:ok, %{scope: scope}} + ) + when scope != :root and transaction_ref in ["next", "previous", "transaction", "contract"] and + transaction_field in @transaction_fields do + {node, acc} + end + + def prewalk( + node = {{:atom, _}, {{:., _, [{{:atom, transaction_ref}, _, nil}, {:atom, type}]}, _, _}}, + acc + ) + when transaction_ref in ["next", "previous", "transaction", "contract"] and + type in @transaction_fields do + {node, acc} + end + + # Whitelist the size/1 function + def prewalk( + node = {{:atom, "size"}, _, [_data]}, + acc = {:ok, %{scope: scope}} + ) + when scope != :root do + {node, acc} + end + + # Whitelist the hash/1 function + def prewalk(node = {{:atom, "hash"}, _, [_data]}, acc = {:ok, %{scope: scope}}) + when scope != :root, + do: {node, acc} + + # Whitelist the regex_match?/2 function + def prewalk( + node = {{:atom, "regex_match?"}, _, [_input, _search]}, + acc = {:ok, %{scope: scope}} + ) + when scope != :root, + do: {node, acc} + + # Whitelist the regex_extract/2 function + def prewalk( + node = {{:atom, "regex_extract"}, _, [_input, _search]}, + acc = {:ok, %{scope: scope}} + ) + when scope != :root, + do: {node, acc} + + # Whitelist the json_path_extract/2 function + def prewalk( + node = {{:atom, "json_path_extract"}, _, [_input, _search]}, + acc = {:ok, %{scope: scope}} + ) + when scope != :root, + do: {node, acc} + + # Whitelist the json_path_match?/2 function + def prewalk( + node = {{:atom, "json_path_match?"}, _, [_input, _search]}, + acc = {:ok, %{scope: scope}} + ) + when scope != :root, + do: {node, acc} + + # Whitelist the get_genesis_address/1 function + def prewalk( + node = {{:atom, "get_genesis_address"}, _, [_address]}, + acc = {:ok, %{scope: scope}} + ) + when scope != :root do + {node, acc} + end + + # Whitelist the get_genesis_public_key/1 function + def prewalk( + node = {{:atom, "get_genesis_public_key"}, _, [_address]}, + acc = {:ok, %{scope: scope}} + ) + when scope != :root do + {node, acc} + end + + # Whitelist the timestamp/0 function in condition + def prewalk(node = {{:atom, "timestamp"}, _, _}, acc = {:ok, %{scope: scope}}) + when scope != :root do + {node, acc} + end + + # Blacklist everything else + def prewalk(node, _acc) do + throw({:error, node}) + end + + @spec postwalk(Macro.t(), any()) :: {Macro.t(), any()} + def postwalk(node = {{:atom, fun}, _, _}, {:ok, context = %{scope: {:function, _, scope}}}) + when fun in @library_functions_names or fun in @transaction_statements_functions_names do + {node, {:ok, %{context | scope: scope}}} + end + + def postwalk( + {{:., meta1, [Access, :get]}, meta2, + [{{:., meta3, [{subject, meta4, nil}, {:atom, field}]}, meta5, []}, {:atom, key}]}, + acc = {:ok, _} + ) do + { + {{:., meta1, [Access, :get]}, meta2, + [ + {{:., meta3, [{subject, meta4, nil}, String.to_existing_atom(field)]}, meta5, []}, + Base.decode16!(key, case: :mixed) + ]}, + acc + } + end + + # Convert map key to binary + def postwalk({:%{}, meta, params}, acc = {:ok, _}) do + encoded_params = + Enum.map(params, fn + {{:atom, key}, value} when is_binary(key) -> + case Base.decode16(key, case: :mixed) do + {:ok, bin} -> + {bin, value} + + :error -> + {key, value} + end + + {key, value} -> + {key, value} + end) + + {{:%{}, meta, encoded_params}, acc} + end + + def postwalk(node, acc) when is_binary(node) do + if String.printable?(node) do + case Base.decode16(node, case: :mixed) do + {:ok, hex} -> + {Base.encode16(hex), acc} + + _ -> + {node, acc} + end + else + {node, acc} + end + end + + def postwalk(node, acc), do: {node, acc} + + @doc """ + Inject context variables and functions by transforming the ast + """ + @spec inject_bindings_and_functions(Macro.t(), list()) :: Macro.t() + def inject_bindings_and_functions(quoted_code, opts) when is_list(opts) do + bindings = Keyword.get(opts, :bindings, %{}) + subject = Keyword.get(opts, :subject) + + {ast, _} = + Macro.postwalk( + quoted_code, + %{ + bindings: bindings, + library_functions: @library_functions_names, + transaction_statements_functions: @transaction_statements_functions_names, + subject: subject + }, + &do_postwalk_execution/2 + ) + + ast + end + + defp do_postwalk_execution({:=, metadata, [var_name, content]}, acc) do + put_ast = + {{:., metadata, [{:__aliases__, metadata, [:Map]}, :put]}, metadata, + [{:scope, metadata, nil}, var_name, content]} + + { + {:=, metadata, [{:scope, metadata, nil}, put_ast]}, + put_in(acc, [:bindings, var_name], content) + } + end + + defp do_postwalk_execution(_node = {{:atom, atom}, metadata, args}, acc) + when atom in @library_functions_names do + fun = Map.get(@library_functions_names_atoms, atom) + + {{{:., metadata, [{:__aliases__, [alias: Library], [:Library]}, fun]}, metadata, args}, acc} + end + + defp do_postwalk_execution(_node = {{:atom, atom}, metadata, args}, acc) + when atom in @transaction_statements_functions_names do + args = + Enum.map(args, fn arg -> + {ast, _} = Macro.postwalk(arg, acc, &do_postwalk_execution/2) + ast + end) + + fun = Map.get(@transaction_statements_functions_names_atoms, atom) + + ast = { + {:., metadata, + [ + {:__aliases__, [alias: TransactionStatements], [:TransactionStatements]}, + fun + ]}, + metadata, + [{:&, metadata, [1]} | args] + } + + update_ast = + {:update_in, metadata, + [ + {:scope, metadata, nil}, + ["next_transaction"], + {:&, metadata, + [ + ast + ]} + ]} + + { + {:=, metadata, [{:scope, metadata, nil}, update_ast]}, + acc + } + end + + defp do_postwalk_execution( + _node = {{:atom, atom}, metadata, _args}, + acc = %{bindings: bindings, subject: subject} + ) do + if Map.has_key?(bindings, atom) do + search = + case subject do + nil -> + [atom] + + subject -> + # Do not use the subject when using reserved keyword + if atom in ["contract", "transaction", "next", "previous"] do + [atom] + else + [subject, atom] + end + end + + { + {:get_in, metadata, [{:scope, metadata, nil}, search]}, + acc + } + else + {atom, acc} + end + end + + defp do_postwalk_execution({:., metadata, [parent, {{:atom, field}}]}, acc) do + { + {:get_in, metadata, [{:scope, metadata, nil}, [parent, field]]}, + acc + } + end + + defp do_postwalk_execution( + {:., _, [{:get_in, metadata, [{:scope, _, nil}, access]}, {:atom, field}]}, + acc + ) do + { + {:get_in, metadata, [{:scope, metadata, nil}, access ++ [field]]}, + acc + } + end + + defp do_postwalk_execution( + {:., metadata, + [{:get_in, metadata, [{:scope, metadata, nil}, [parent]]}, {:atom, child}]}, + acc + ) do + {{:get_in, metadata, [{:scope, metadata, nil}, [parent, child]]}, acc} + end + + defp do_postwalk_execution({{:atom, atom}, val}, acc) do + {ast, _} = Macro.postwalk(val, acc, &do_postwalk_execution/2) + {{atom, ast}, acc} + end + + defp do_postwalk_execution({{:get_in, metadata, [{:scope, metadata, nil}, access]}, _, []}, acc) do + { + {:get_in, metadata, [{:scope, metadata, nil}, access]}, + acc + } + end + + defp do_postwalk_execution({:==, _, [{left, _, args_l}, {right, _, args_r}]}, acc) do + {{:==, [], [{left, [], args_l}, {right, [], args_r}]}, acc} + end + + defp do_postwalk_execution({:==, _, [{left, _, args}, right]}, acc) do + {{:==, [], [{left, [], args}, right]}, acc} + end + + defp do_postwalk_execution({node, _, _}, acc) when is_binary(node) do + {node, acc} + end + + defp do_postwalk_execution(node, acc), do: {node, acc} + + @doc """ + Format an error message from the failing ast node + + It returns message with metadata if possible to indicate the line of the error + """ + @spec format_error_reason(any(), String.t()) :: String.t() + def format_error_reason({:atom, _key}, reason) do + do_format_error_reason(reason, "", []) + end + + def format_error_reason({{:atom, key}, metadata, _}, reason) do + do_format_error_reason(reason, key, metadata) + end + + def format_error_reason({_, metadata, [{:__aliases__, _, [atom: module]} | _]}, reason) do + do_format_error_reason(reason, module, metadata) + end + + def format_error_reason(ast_node = {_, metadata, _}, reason) do + do_format_error_reason(reason, Macro.to_string(ast_node), metadata) + end + + def format_error_reason({{:atom, _}, {_, metadata, _}}, reason) do + do_format_error_reason(reason, "", metadata) + end + + def format_error_reason({{:atom, key}, _}, reason) do + do_format_error_reason(reason, key, []) + end + + defp do_format_error_reason(message, cause, metadata) do + message = prepare_message(message) + + [prepare_message(message), cause, metadata_to_string(metadata)] + |> Enum.reject(&(&1 == "")) + |> Enum.join(" - ") + end + + defp prepare_message(message) when is_atom(message) do + message |> Atom.to_string() |> String.replace("_", " ") + end + + defp prepare_message(message) when is_binary(message) do + String.trim_trailing(message, ":") + end + + defp metadata_to_string(line: line, column: column), do: "L#{line}:C#{column}" + defp metadata_to_string(line: line), do: "L#{line}" + defp metadata_to_string(_), do: "" +end diff --git a/lib/archethic/contracts/loader.ex b/lib/archethic/contracts/loader.ex index 29247d0956..0ce8d0f313 100644 --- a/lib/archethic/contracts/loader.ex +++ b/lib/archethic/contracts/loader.ex @@ -11,6 +11,7 @@ defmodule Archethic.Contracts.Loader do alias Archethic.DB + alias Archethic.TransactionChain alias Archethic.TransactionChain.Transaction alias Archethic.TransactionChain.Transaction.ValidationStamp alias Archethic.TransactionChain.TransactionData @@ -63,27 +64,23 @@ defmodule Archethic.Contracts.Loader do when code != "" do stop_contract(Transaction.previous_address(tx)) - case Contracts.parse!(code) do - # Only load smart contract which are expecting interactions - %Contract{triggers: triggers = [_ | _]} -> - triggers = Enum.reject(triggers, &(&1.actions == {:__block__, [], []})) - - # Avoid to load empty smart contract - if length(triggers) > 0 do - {:ok, _} = - DynamicSupervisor.start_child( - ContractSupervisor, - {Worker, Contract.from_transaction!(tx)} - ) - - Logger.info("Smart contract loaded", - transaction_address: Base.encode16(address), - transaction_type: type - ) - end + %Contract{triggers: triggers} = Contracts.parse!(code) + triggers = Enum.reject(triggers, fn {_, actions} -> actions == {:__block__, [], []} end) - _ -> - :ok + # Create worker only load smart contract which are expecting interactions and where the actions are not empty + if length(triggers) > 0 do + {:ok, _} = + DynamicSupervisor.start_child( + ContractSupervisor, + {Worker, Contract.from_transaction!(tx)} + ) + + Logger.info("Smart contract loaded", + transaction_address: Base.encode16(address), + transaction_type: type + ) + else + :ok end end @@ -91,7 +88,11 @@ defmodule Archethic.Contracts.Loader do tx = %Transaction{ address: tx_address, type: tx_type, - validation_stamp: %ValidationStamp{timestamp: tx_timestamp, recipients: recipients} + validation_stamp: %ValidationStamp{ + timestamp: tx_timestamp, + recipients: recipients, + protocol_version: protocol_version + } }, false ) @@ -104,7 +105,12 @@ defmodule Archethic.Contracts.Loader do case Worker.execute(contract_address, tx) do :ok -> - TransactionLookup.add_contract_transaction(contract_address, tx_address, tx_timestamp) + TransactionLookup.add_contract_transaction( + contract_address, + tx_address, + tx_timestamp, + protocol_version + ) Logger.info("Transaction towards contract ingested", transaction_address: Base.encode16(tx_address), @@ -121,12 +127,19 @@ defmodule Archethic.Contracts.Loader do %Transaction{ address: address, type: type, - validation_stamp: %ValidationStamp{recipients: recipients, timestamp: timestamp} + validation_stamp: %ValidationStamp{ + recipients: recipients, + timestamp: timestamp, + protocol_version: protocol_version + } }, true ) when recipients != [] do - Enum.each(recipients, &TransactionLookup.add_contract_transaction(&1, address, timestamp)) + Enum.each( + recipients, + &TransactionLookup.add_contract_transaction(&1, address, timestamp, protocol_version) + ) Logger.info("Transaction towards contract ingested", transaction_address: Base.encode16(address), @@ -144,6 +157,8 @@ defmodule Archethic.Contracts.Loader do case Registry.lookup(ContractRegistry, address) do [{pid, _}] -> DynamicSupervisor.terminate_child(ContractSupervisor, pid) + TransactionLookup.clear_contract_transactions(address) + TransactionChain.clear_pending_transactions(address) Logger.info("Stop smart contract at #{Base.encode16(address)}") _ -> diff --git a/lib/archethic/contracts/transaction_lookup.ex b/lib/archethic/contracts/transaction_lookup.ex index bafbc8bfc2..14b37c7b8e 100644 --- a/lib/archethic/contracts/transaction_lookup.ex +++ b/lib/archethic/contracts/transaction_lookup.ex @@ -6,6 +6,10 @@ defmodule Archethic.Contracts.TransactionLookup do use GenServer @vsn Mix.Project.config()[:version] + alias Archethic.DB + alias Archethic.TransactionChain.TransactionInput + alias Archethic.TransactionChain.VersionedTransactionInput + require Logger @doc """ @@ -29,11 +33,24 @@ defmodule Archethic.Contracts.TransactionLookup do @doc """ Register a new transaction address towards a contract address """ - @spec add_contract_transaction(binary(), binary(), DateTime.t()) :: :ok - def add_contract_transaction(contract_address, transaction_address, transaction_timestamp) + @spec add_contract_transaction( + contract_address :: binary(), + tx_address :: binary(), + tx_timestamp :: DateTime.t(), + protocol_version :: pos_integer() + ) :: :ok + def add_contract_transaction( + contract_address, + transaction_address, + transaction_timestamp, + protocol_version + ) when is_binary(contract_address) and is_binary(transaction_address) do true = - :ets.insert(@table_name, {contract_address, transaction_address, transaction_timestamp}) + :ets.insert( + @table_name, + {contract_address, transaction_address, transaction_timestamp, protocol_version} + ) :ok end @@ -41,10 +58,53 @@ defmodule Archethic.Contracts.TransactionLookup do @doc """ Return the list transaction towards a contract address """ - @spec list_contract_transactions(binary()) :: list({binary(), DateTime.t()}) + @spec list_contract_transactions(binary()) :: + list( + {address :: binary(), timestamp :: DateTime.t(), protocol_version :: pos_integer()} + ) def list_contract_transactions(contract_address) when is_binary(contract_address) do - Enum.map(:ets.lookup(@table_name, contract_address), fn {_, tx_address, tx_timestamp} -> - {tx_address, tx_timestamp} + case :ets.lookup(@table_name, contract_address) do + [] -> + DB.get_inputs(:call, contract_address) + |> Enum.map(fn %VersionedTransactionInput{ + input: %TransactionInput{from: from, timestamp: timestamp}, + protocol_version: protocol_version + } -> + {from, timestamp, protocol_version} + end) + + inputs -> + Enum.map(inputs, fn {_, tx_address, tx_timestamp, protocol_version} -> + {tx_address, tx_timestamp, protocol_version} + end) + end + end + + @doc """ + Remove the contract transactions + """ + @spec clear_contract_transactions(binary()) :: :ok + def clear_contract_transactions(contract_address) when is_binary(contract_address) do + {:ok, pid} = DB.start_inputs_writer(:call, contract_address) + + contract_address + |> list_contract_transactions() + |> Enum.each(fn {tx_address, tx_timestamp, protocol_version} -> + input = %VersionedTransactionInput{ + input: %TransactionInput{ + from: tx_address, + timestamp: tx_timestamp, + type: :call + }, + protocol_version: protocol_version + } + + DB.append_input(pid, input) end) + + DB.stop_inputs_writer(pid) + + :ets.delete(@table_name, contract_address) + :ok end end diff --git a/lib/archethic/contracts/worker.ex b/lib/archethic/contracts/worker.ex index 3ee05903c5..eed5630fac 100644 --- a/lib/archethic/contracts/worker.ex +++ b/lib/archethic/contracts/worker.ex @@ -5,10 +5,10 @@ defmodule Archethic.Contracts.Worker do alias Archethic.ContractRegistry alias Archethic.Contracts.Contract - alias Archethic.Contracts.Contract.Conditions - alias Archethic.Contracts.Contract.Constants - alias Archethic.Contracts.Contract.Trigger - alias Archethic.Contracts.Interpreter + alias Archethic.Contracts.ContractConditions, as: Conditions + alias Archethic.Contracts.ContractConstants, as: Constants + alias Archethic.Contracts.ActionInterpreter + alias Archethic.Contracts.ConditionInterpreter alias Archethic.Crypto @@ -19,7 +19,6 @@ defmodule Archethic.Contracts.Worker do alias Archethic.OracleChain alias Archethic.P2P - alias Archethic.P2P.Message.StartMining alias Archethic.P2P.Node alias Archethic.PubSub @@ -62,10 +61,10 @@ defmodule Archethic.Contracts.Worker do def handle_continue(:start_schedulers, state = %{contract: %Contract{triggers: triggers}}) do new_state = - Enum.reduce(triggers, state, fn trigger = %Trigger{type: type}, acc -> - case schedule_trigger(trigger) do + Enum.reduce(triggers, state, fn {trigger_type, _}, acc -> + case schedule_trigger(trigger_type) do timer when is_reference(timer) -> - Map.update(acc, :timers, %{type => timer}, &Map.put(&1, type, timer)) + Map.update(acc, :timers, %{trigger_type => timer}, &Map.put(&1, trigger_type, timer)) _ -> acc @@ -79,14 +78,13 @@ defmodule Archethic.Contracts.Worker do {:execute, incoming_tx = %Transaction{}}, _from, state = %{ - contract: - contract = %Contract{ - triggers: triggers, - constants: %Constants{ - contract: contract_constants = %{"address" => contract_address} - }, - conditions: %{transaction: transaction_condition} - } + contract: %Contract{ + triggers: triggers, + constants: %Constants{ + contract: contract_constants = %{"address" => contract_address} + }, + conditions: %{transaction: transaction_condition} + } } ) do Logger.info("Execute contract transaction actions", @@ -95,7 +93,7 @@ defmodule Archethic.Contracts.Worker do contract: Base.encode16(contract_address) ) - if Enum.any?(triggers, &(&1.type == :transaction)) do + if Map.has_key?(triggers, :transaction) do constants = %{ "contract" => contract_constants, "transaction" => Constants.from_transaction(incoming_tx) @@ -103,8 +101,8 @@ defmodule Archethic.Contracts.Worker do contract_transaction = Constants.to_transaction(contract_constants) - with true <- Interpreter.valid_conditions?(transaction_condition, constants), - next_tx <- Interpreter.execute_actions(contract, :transaction, constants), + with true <- ConditionInterpreter.valid_conditions?(transaction_condition, constants), + next_tx <- ActionInterpreter.execute(Map.fetch!(triggers, :transaction), constants), {:ok, next_tx} <- chain_transaction(next_tx, contract_transaction) do handle_new_transaction(next_tx) {:reply, :ok, state} @@ -133,13 +131,12 @@ defmodule Archethic.Contracts.Worker do end def handle_info( - %Trigger{type: :datetime}, + {:trigger, {:datetime, timestamp}}, state = %{ - contract: - contract = %Contract{ - constants: %Constants{contract: contract_constants = %{"address" => address}} - }, - timers: %{datetime: timer} + contract: %Contract{ + triggers: triggers, + constants: %Constants{contract: contract_constants = %{"address" => address}} + } } ) do Logger.info("Execute contract datetime trigger actions", @@ -152,24 +149,24 @@ defmodule Archethic.Contracts.Worker do contract_tx = Constants.to_transaction(contract_constants) - with next_tx <- Interpreter.execute_actions(contract, :datetime, constants), + with next_tx <- + ActionInterpreter.execute(Map.fetch!(triggers, {:datetime, timestamp}), constants), {:ok, next_tx} <- chain_transaction(next_tx, contract_tx) do handle_new_transaction(next_tx) end - Process.cancel_timer(timer) - {:noreply, Map.update!(state, :timers, &Map.delete(&1, :datetime))} + {:noreply, Map.update!(state, :timers, &Map.delete(&1, {:datetime, timestamp}))} end def handle_info( - trigger = %Trigger{type: :interval}, + {:trigger, {:interval, interval}}, state = %{ - contract: - contract = %Contract{ - constants: %Constants{ - contract: contract_constants = %{"address" => address} - } + contract: %Contract{ + triggers: triggers, + constants: %Constants{ + contract: contract_constants = %{"address" => address} } + } } ) do Logger.info("Execute contract interval trigger actions", @@ -177,7 +174,7 @@ defmodule Archethic.Contracts.Worker do ) # Schedule the next interval trigger - interval_timer = schedule_trigger(trigger) + interval_timer = schedule_trigger({:interval, interval}) constants = %{ "contract" => contract_constants @@ -186,7 +183,8 @@ defmodule Archethic.Contracts.Worker do contract_transaction = Constants.to_transaction(contract_constants) with true <- enough_funds?(address), - next_tx <- Interpreter.execute_actions(contract, :interval, constants), + next_tx <- + ActionInterpreter.execute(Map.fetch!(triggers, {:interval, interval}), constants), {:ok, next_tx} <- chain_transaction(next_tx, contract_transaction), :ok <- ensure_enough_funds(next_tx, address) do handle_new_transaction(next_tx) @@ -198,18 +196,17 @@ defmodule Archethic.Contracts.Worker do def handle_info( {:new_transaction, tx_address, :oracle, _timestamp}, state = %{ - contract: - contract = %Contract{ - triggers: triggers, - constants: %Constants{contract: contract_constants = %{"address" => address}}, - conditions: %{oracle: oracle_condition} - } + contract: %Contract{ + triggers: triggers, + constants: %Constants{contract: contract_constants = %{"address" => address}}, + conditions: %{oracle: oracle_condition} + } } ) do Logger.info("Execute contract oracle trigger actions", contract: Base.encode16(address)) with true <- enough_funds?(address), - true <- Enum.any?(triggers, &(&1.type == :oracle)) do + true <- Map.has_key?(triggers, :oracle) do {:ok, tx} = TransactionChain.get_transaction(tx_address) constants = %{ @@ -220,14 +217,14 @@ defmodule Archethic.Contracts.Worker do contract_transaction = Constants.to_transaction(contract_constants) if Conditions.empty?(oracle_condition) do - with next_tx <- Interpreter.execute_actions(contract, :oracle, constants), + with next_tx <- ActionInterpreter.execute(Map.fetch!(triggers, :oracle), constants), {:ok, next_tx} <- chain_transaction(next_tx, contract_transaction), :ok <- ensure_enough_funds(next_tx, address) do handle_new_transaction(next_tx) end else - with true <- Interpreter.valid_conditions?(oracle_condition, constants), - next_tx <- Interpreter.execute_actions(contract, :oracle, constants), + with true <- ConditionInterpreter.valid_conditions?(oracle_condition, constants), + next_tx <- ActionInterpreter.execute(Map.fetch!(triggers, :oracle), constants), {:ok, next_tx} <- chain_transaction(next_tx, contract_transaction), :ok <- ensure_enough_funds(next_tx, address) do handle_new_transaction(next_tx) @@ -244,39 +241,40 @@ defmodule Archethic.Contracts.Worker do {:noreply, state} end - def handle_info({:EXIT, _pid, _}, _state) do - :keep_state_and_data + def handle_info({:EXIT, _pid, _}, state) do + {:noreply, state} end defp via_tuple(address) do {:via, Registry, {ContractRegistry, address}} end - defp schedule_trigger( - trigger = %Trigger{ - type: :interval, - opts: opts - } - ) do - interval = Keyword.fetch!(opts, :at) - enable_seconds? = Keyword.get(opts, :enable_seconds, false) + # Only used for testing + defp schedule_trigger(trigger = {:interval, {interval, :second}}) do + Process.send_after( + self(), + {:trigger, trigger}, + Utils.time_offset(interval, DateTime.utc_now(), true) * 1000 + ) + end + defp schedule_trigger(trigger = {:interval, interval}) do Process.send_after( self(), - trigger, - Utils.time_offset(interval, DateTime.utc_now(), enable_seconds?) * 1000 + {:trigger, trigger}, + Utils.time_offset(interval, DateTime.utc_now()) * 1000 ) end - defp schedule_trigger(trigger = %Trigger{type: :datetime, opts: [at: datetime = %DateTime{}]}) do + defp schedule_trigger(trigger = {:datetime, datetime = %DateTime{}}) do seconds = DateTime.diff(datetime, DateTime.utc_now()) if seconds > 0 do - Process.send_after(self(), trigger, seconds * 1000) + Process.send_after(self(), {:trigger, trigger}, seconds * 1000) end end - defp schedule_trigger(%Trigger{type: :oracle}) do + defp schedule_trigger(:oracle) do PubSub.register_to_new_transaction_by_type(:oracle) end @@ -287,11 +285,7 @@ defmodule Archethic.Contracts.Worker do # The first storage node of the contract initiate the sending of the new transaction if trigger_node?(validation_nodes) do - P2P.broadcast_message(validation_nodes, %StartMining{ - transaction: next_transaction, - validation_node_public_keys: Enum.map(validation_nodes, & &1.last_public_key), - welcome_node_public_key: Crypto.last_node_public_key() - }) + Archethic.send_new_transaction(next_transaction) else DetectNodeResponsiveness.start_link( next_transaction.address, @@ -300,11 +294,7 @@ defmodule Archethic.Contracts.Worker do Logger.info("contract transaction ...attempt #{count}") if trigger_node?(validation_nodes, count) do - P2P.broadcast_message(validation_nodes, %StartMining{ - transaction: next_transaction, - validation_node_public_keys: Enum.map(validation_nodes, & &1.last_public_key), - welcome_node_public_key: Crypto.last_node_public_key() - }) + Archethic.send_new_transaction(next_transaction) end end ) diff --git a/lib/archethic/db.ex b/lib/archethic/db.ex index ee5efd4e14..8c38694096 100644 --- a/lib/archethic/db.ex +++ b/lib/archethic/db.ex @@ -7,6 +7,7 @@ defmodule Archethic.DB do alias Archethic.Crypto alias __MODULE__.EmbeddedImpl + alias __MODULE__.EmbeddedImpl.InputsWriter alias Archethic.TransactionChain.Transaction alias Archethic.TransactionChain.VersionedTransactionInput @@ -76,11 +77,12 @@ defmodule Archethic.DB do @callback get_bootstrap_info(key :: String.t()) :: String.t() | nil @callback set_bootstrap_info(key :: String.t(), value :: String.t()) :: :ok - @callback start_inputs_writer(ledger :: :UCO | :token, address :: binary()) :: {:ok, pid()} + @callback start_inputs_writer(input_type :: InputsWriter.input_type(), address :: binary()) :: + {:ok, pid()} @callback stop_inputs_writer(pid :: pid()) :: :ok @callback append_input(pid :: pid(), VersionedTransactionInput.t()) :: :ok - @callback get_inputs(ledger :: :UCO | :token, address :: binary()) :: + @callback get_inputs(input_type :: InputsWriter.input_type(), address :: binary()) :: list(VersionedTransactionInput.t()) @callback stream_first_addresses() :: Enumerable.t() diff --git a/lib/archethic/db/embedded_impl.ex b/lib/archethic/db/embedded_impl.ex index 91800661c0..99286962e6 100644 --- a/lib/archethic/db/embedded_impl.ex +++ b/lib/archethic/db/embedded_impl.ex @@ -388,8 +388,9 @@ defmodule Archethic.DB.EmbeddedImpl do @doc """ Start a process responsible to write the inputs """ - @spec start_inputs_writer(ledger :: :UCO | :token, address :: binary()) :: {:ok, pid()} - defdelegate start_inputs_writer(ledger, address), to: InputsWriter, as: :start_link + @spec start_inputs_writer(input_type :: InputsWriter.input_type(), address :: binary()) :: + {:ok, pid()} + defdelegate start_inputs_writer(input_type, address), to: InputsWriter, as: :start_link @doc """ Stop the process responsible to write the inputs @@ -407,7 +408,7 @@ defmodule Archethic.DB.EmbeddedImpl do @doc """ Read the list of inputs available at address """ - @spec get_inputs(ledger :: :UCO | :token, address :: binary()) :: + @spec get_inputs(input_type :: InputsWriter.input_type(), address :: binary()) :: list(VersionedTransactionInput.t()) defdelegate get_inputs(ledger, address), to: InputsReader, as: :get_inputs diff --git a/lib/archethic/db/embedded_impl/inputs_reader.ex b/lib/archethic/db/embedded_impl/inputs_reader.ex index 12f269a20f..51167a2722 100644 --- a/lib/archethic/db/embedded_impl/inputs_reader.ex +++ b/lib/archethic/db/embedded_impl/inputs_reader.ex @@ -7,7 +7,7 @@ defmodule Archethic.DB.EmbeddedImpl.InputsReader do alias Archethic.TransactionChain.VersionedTransactionInput alias Archethic.Utils - @spec get_inputs(ledger :: InputsWriter.ledger(), address :: binary()) :: + @spec get_inputs(input_type :: InputsWriter.input_type(), address :: binary()) :: list(VersionedTransactionInput.t()) def get_inputs(ledger, address) do filename = InputsWriter.address_to_filename(ledger, address) diff --git a/lib/archethic/db/embedded_impl/inputs_writer.ex b/lib/archethic/db/embedded_impl/inputs_writer.ex index 3a909b94cf..df10a7febf 100644 --- a/lib/archethic/db/embedded_impl/inputs_writer.ex +++ b/lib/archethic/db/embedded_impl/inputs_writer.ex @@ -9,9 +9,10 @@ defmodule Archethic.DB.EmbeddedImpl.InputsWriter do alias Archethic.TransactionChain.VersionedTransactionInput alias Archethic.Utils - @type ledger :: :token | :UCO + @type input_type :: :UCO | :token | :call - @spec start_link(ledger :: ledger, address :: binary()) :: {:ok, pid()} + @spec start_link(type :: input_type(), address :: binary()) :: + {:ok, pid()} def start_link(ledger, address) do GenServer.start_link(__MODULE__, ledger: ledger, address: address) end @@ -29,12 +30,14 @@ defmodule Archethic.DB.EmbeddedImpl.InputsWriter do GenServer.call(pid, {:append_input, input}) end - @spec address_to_filename(ledger :: ledger, address :: binary()) :: String.t() + @spec address_to_filename(type :: input_type(), address :: binary()) :: + String.t() def address_to_filename(ledger, address) do prefix = case ledger do :UCO -> "uco" :token -> "token" + :call -> "call" end Path.join([EmbeddedImpl.db_path(), "inputs", prefix, Base.encode16(address)]) diff --git a/lib/archethic/mining/pending_transaction_validation.ex b/lib/archethic/mining/pending_transaction_validation.ex index 7485aa8929..a66fa674d6 100644 --- a/lib/archethic/mining/pending_transaction_validation.ex +++ b/lib/archethic/mining/pending_transaction_validation.ex @@ -225,7 +225,7 @@ defmodule Archethic.Mining.PendingTransactionValidation do data: %TransactionData{code: code, ownerships: ownerships} }) do case Contracts.parse(code) do - {:ok, %Contract{triggers: [_ | _]}} -> + {:ok, %Contract{triggers: triggers}} when map_size(triggers) > 0 -> if Enum.any?( ownerships, &Ownership.authorized_public_key?(&1, Crypto.storage_nonce_public_key()) diff --git a/lib/archethic/mining/proof_of_work.ex b/lib/archethic/mining/proof_of_work.ex index a162fb539f..17e6785823 100644 --- a/lib/archethic/mining/proof_of_work.ex +++ b/lib/archethic/mining/proof_of_work.ex @@ -11,7 +11,7 @@ defmodule Archethic.Mining.ProofOfWork do alias Archethic.Contracts alias Archethic.Contracts.Contract - alias Archethic.Contracts.Contract.Conditions + alias Archethic.Contracts.ContractConditions alias Archethic.Crypto @@ -117,14 +117,13 @@ defmodule Archethic.Mining.ProofOfWork do @spec list_origin_public_keys_candidates(Transaction.t()) :: list(Crypto.key()) def list_origin_public_keys_candidates(tx = %Transaction{data: %TransactionData{code: code}}) when code != "" do - %Contract{conditions: %{inherit: %Conditions{origin_family: family}}} = Contracts.parse!(code) + case Contracts.parse(code) do + {:ok, %Contract{conditions: %{inherit: %ContractConditions{origin_family: family}}}} + when family != :all -> + SharedSecrets.list_origin_public_keys(family) - case family do - :all -> + _ -> do_list_origin_public_keys_candidates(tx) - - family -> - SharedSecrets.list_origin_public_keys(family) end end diff --git a/lib/archethic/p2p/message.ex b/lib/archethic/p2p/message.ex index 4d65a69563..f79fc15ea1 100644 --- a/lib/archethic/p2p/message.ex +++ b/lib/archethic/p2p/message.ex @@ -1633,8 +1633,11 @@ defmodule Archethic.P2P.Message do contract_inputs = address |> Contracts.list_contract_transactions() - |> Enum.map(fn {address, timestamp} -> - %TransactionInput{from: address, type: :call, timestamp: timestamp} + |> Enum.map(fn {address, timestamp, protocol_version} -> + %VersionedTransactionInput{ + input: %TransactionInput{from: address, type: :call, timestamp: timestamp}, + protocol_version: protocol_version + } end) inputs = Account.get_inputs(address) ++ contract_inputs diff --git a/lib/archethic/transaction_chain.ex b/lib/archethic/transaction_chain.ex index 3bbd9875f9..03cd11f295 100644 --- a/lib/archethic/transaction_chain.ex +++ b/lib/archethic/transaction_chain.ex @@ -251,6 +251,12 @@ defmodule Archethic.TransactionChain do @spec pending_transaction_signed_by?(to :: binary(), from :: binary()) :: boolean() defdelegate pending_transaction_signed_by?(to, from), to: PendingLedger, as: :already_signed? + @doc """ + Clear the transactions stored as pending + """ + @spec clear_pending_transactions(binary()) :: :ok + defdelegate clear_pending_transactions(address), to: PendingLedger, as: :remove_address + @doc """ Determine if the transaction exists """ diff --git a/lib/archethic/transaction_chain/mem_tables_loader.ex b/lib/archethic/transaction_chain/mem_tables_loader.ex index 97b7021e36..44e48f890e 100644 --- a/lib/archethic/transaction_chain/mem_tables_loader.ex +++ b/lib/archethic/transaction_chain/mem_tables_loader.ex @@ -5,7 +5,7 @@ defmodule Archethic.TransactionChain.MemTablesLoader do @vsn Mix.Project.config()[:version] alias Archethic.Contracts.Contract - alias Archethic.Contracts.Contract.Conditions + alias Archethic.Contracts.ContractConditions alias Archethic.DB @@ -70,7 +70,7 @@ defmodule Archethic.TransactionChain.MemTablesLoader do %Contract{conditions: %{transaction: transaction_conditions}} = Contract.from_transaction!(tx) # TODO: improve the criteria of pending detection - if Conditions.empty?(transaction_conditions) do + if ContractConditions.empty?(transaction_conditions) do :ok else PendingLedger.add_address(address) diff --git a/lib/archethic/transaction_chain/transaction/validation_stamp.ex b/lib/archethic/transaction_chain/transaction/validation_stamp.ex index 1bc9fc4b40..b5e6cf25e6 100755 --- a/lib/archethic/transaction_chain/transaction/validation_stamp.ex +++ b/lib/archethic/transaction_chain/transaction/validation_stamp.ex @@ -4,6 +4,8 @@ defmodule Archethic.TransactionChain.Transaction.ValidationStamp do """ alias Archethic.Crypto + + alias Archethic.Utils alias Archethic.Utils.VarInt alias __MODULE__.LedgerOperations @@ -275,7 +277,7 @@ defmodule Archethic.TransactionChain.Transaction.ValidationStamp do {recipients_length, rest} = rest |> VarInt.get_value() {recipients, <>} = - deserialize_list_of_recipients_addresses(rest, recipients_length, []) + Utils.deserialize_addresses(rest, recipients_length, []) error = deserialize_error(error_byte) @@ -403,26 +405,6 @@ defmodule Archethic.TransactionChain.Transaction.ValidationStamp do Crypto.verify?(signature, raw_stamp, public_key) end - defp deserialize_list_of_recipients_addresses(rest, 0, _acc), do: {[], rest} - - defp deserialize_list_of_recipients_addresses(rest, nb_recipients, acc) - when length(acc) == nb_recipients do - {Enum.reverse(acc), rest} - end - - defp deserialize_list_of_recipients_addresses( - <>, - nb_recipients, - acc - ) do - hash_size = Crypto.hash_size(hash_id) - <> = rest - - deserialize_list_of_recipients_addresses(rest, nb_recipients, [ - <> | acc - ]) - end - defp serialize_error(nil), do: 0 defp serialize_error(:invalid_pending_transaction), do: 1 defp serialize_error(:invalid_inherit_constraints), do: 2 diff --git a/lib/archethic/transaction_chain/transaction_input.ex b/lib/archethic/transaction_chain/transaction_input.ex index 0c41930dae..b29ce3d9c6 100644 --- a/lib/archethic/transaction_chain/transaction_input.ex +++ b/lib/archethic/transaction_chain/transaction_input.ex @@ -13,7 +13,7 @@ defmodule Archethic.TransactionChain.TransactionInput do @type t() :: %__MODULE__{ from: Crypto.versioned_hash(), - amount: pos_integer(), + amount: pos_integer() | nil, spent?: boolean(), type: TransactionMovementType.t() | :call, timestamp: DateTime.t(), diff --git a/test/archethic/contracts/interpreter/action_test.exs b/test/archethic/contracts/interpreter/action_test.exs new file mode 100644 index 0000000000..f67cc39326 --- /dev/null +++ b/test/archethic/contracts/interpreter/action_test.exs @@ -0,0 +1,346 @@ +defmodule Archethic.Contracts.ActionInterpreterTest do + use ArchethicCase + + alias Archethic.Contracts.ActionInterpreter + alias Archethic.Contracts.Interpreter + + alias Archethic.P2P + alias Archethic.P2P.Node + alias Archethic.P2P.Message.FirstAddress + + alias Archethic.TransactionChain.Transaction + alias Archethic.TransactionChain.TransactionData + + doctest ActionInterpreter + + import Mox + + test "should parse a contract with some standard functions" do + assert {:ok, :transaction, + { + :__block__, + [], + [ + { + :=, + [line: 2], + [ + {:scope, [line: 2], nil}, + { + :update_in, + [line: 2], + [ + {:scope, [line: 2], nil}, + ["next_transaction"], + {:&, [line: 2], + [ + {{:., [line: 2], + [ + {:__aliases__, + [ + alias: Archethic.Contracts.Interpreter.TransactionStatements + ], [:TransactionStatements]}, + :set_type + ]}, [line: 2], [{:&, [line: 2], [1]}, "transfer"]} + ]} + ] + } + ] + }, + { + :=, + [line: 3], + [ + {:scope, [line: 3], nil}, + { + :update_in, + [line: 3], + [ + {:scope, [line: 3], nil}, + ["next_transaction"], + { + :&, + [line: 3], + [ + { + {:., [line: 3], + [ + {:__aliases__, + [ + alias: Archethic.Contracts.Interpreter.TransactionStatements + ], [:TransactionStatements]}, + :add_uco_transfer + ]}, + [line: 3], + [ + {:&, [line: 3], [1]}, + [ + {"to", + "7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3"}, + {"amount", 1_040_000_000} + ] + ] + } + ] + } + ] + } + ] + }, + { + :=, + [line: 4], + [ + {:scope, [line: 4], nil}, + { + :update_in, + [line: 4], + [ + {:scope, [line: 4], nil}, + ["next_transaction"], + { + :&, + [line: 4], + [ + { + {:., [line: 4], + [ + {:__aliases__, + [ + alias: Archethic.Contracts.Interpreter.TransactionStatements + ], [:TransactionStatements]}, + :add_token_transfer + ]}, + [line: 4], + [ + {:&, [line: 4], [1]}, + [ + {"to", + "30670455713E2CBECF94591226A903651ED8625635181DDA236FECC221D1E7E4"}, + {"amount", 20_000_000_000}, + {"token_address", + "AEB4A6F5AB6D82BE223C5867EBA5FE616F52F410DCF83B45AFF158DD40AE8AC3"}, + {"token_id", 0} + ] + ] + } + ] + } + ] + } + ] + }, + { + :=, + [line: 5], + [ + {:scope, [line: 5], nil}, + { + :update_in, + [line: 5], + [ + {:scope, [line: 5], nil}, + ["next_transaction"], + { + :&, + [line: 5], + [ + {{:., [line: 5], + [ + {:__aliases__, + [ + alias: Archethic.Contracts.Interpreter.TransactionStatements + ], [:TransactionStatements]}, + :set_content + ]}, [line: 5], [{:&, [line: 5], [1]}, "Receipt"]} + ] + } + ] + } + ] + }, + { + :=, + [line: 6], + [ + {:scope, [line: 6], nil}, + { + :update_in, + [line: 6], + [ + {:scope, [line: 6], nil}, + ["next_transaction"], + { + :&, + [line: 6], + [ + { + {:., [line: 6], + [ + {:__aliases__, + [ + alias: Archethic.Contracts.Interpreter.TransactionStatements + ], [:TransactionStatements]}, + :add_ownership + ]}, + [line: 6], + [ + {:&, [line: 6], [1]}, + [ + {"secret", "MyEncryptedSecret"}, + {"secret_key", "MySecretKey"}, + {"authorized_public_keys", + [ + "70C245E5D970B59DF65638BDD5D963EE22E6D892EA224D8809D0FB75D0B1907A" + ]} + ] + ] + } + ] + } + ] + } + ] + }, + { + :=, + [line: 7], + [ + {:scope, [line: 7], nil}, + { + :update_in, + [line: 7], + [ + {:scope, [line: 7], nil}, + ["next_transaction"], + { + :&, + [line: 7], + [ + { + {:., [line: 7], + [ + {:__aliases__, + [ + alias: Archethic.Contracts.Interpreter.TransactionStatements + ], [:TransactionStatements]}, + :add_recipient + ]}, + [line: 7], + [ + {:&, [line: 7], [1]}, + "78273C5CBCEB8617F54380CC2F173DF2404DB676C9F10D546B6F395E6F3BDDEE" + ] + } + ] + } + ] + } + ] + } + ] + }} = + """ + actions triggered_by: transaction do + set_type transfer + add_uco_transfer to: \"7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3\", amount: 1040000000 + add_token_transfer to: \"30670455713E2CBECF94591226A903651ED8625635181DDA236FECC221D1E7E4\", amount: 20000000000, token_address: \"AEB4A6F5AB6D82BE223C5867EBA5FE616F52F410DCF83B45AFF158DD40AE8AC3\", token_id: 0 + set_content \"Receipt\" + add_ownership secret: \"MyEncryptedSecret\", secret_key: \"MySecretKey\", authorized_public_keys: ["70C245E5D970B59DF65638BDD5D963EE22E6D892EA224D8809D0FB75D0B1907A"] + add_recipient \"78273C5CBCEB8617F54380CC2F173DF2404DB676C9F10D546B6F395E6F3BDDEE\" + end + """ + |> Interpreter.sanitize_code() + |> elem(1) + |> ActionInterpreter.parse() + end + + test "should evaluate actions based on if statement" do + assert %Transaction{data: %TransactionData{content: "yes"}} = + ~S""" + actions triggered_by: transaction do + if transaction.previous_public_key == "abc" do + set_content "yes" + else + set_content "no" + end + end + """ + |> Interpreter.sanitize_code() + |> elem(1) + |> ActionInterpreter.parse() + |> elem(2) + |> ActionInterpreter.execute(%{"transaction" => %{"previous_public_key" => "abc"}}) + end + + test "should use a variable assignation" do + assert %Transaction{data: %TransactionData{content: "hello"}} = + """ + actions triggered_by: transaction do + new_content = \"hello\" + set_content new_content + end + """ + |> Interpreter.sanitize_code() + |> elem(1) + |> ActionInterpreter.parse() + |> elem(2) + |> ActionInterpreter.execute() + end + + test "should use a interpolation assignation" do + assert %Transaction{data: %TransactionData{content: "hello 4"}} = + ~S""" + actions triggered_by: transaction do + new_content = "hello #{2+2}" + set_content new_content + end + """ + |> Interpreter.sanitize_code() + |> elem(1) + |> ActionInterpreter.parse() + |> elem(2) + |> ActionInterpreter.execute() + end + + test "shall use get_genesis_address/1 in actions" do + key = <<0::16, :crypto.strong_rand_bytes(32)::binary>> + + P2P.add_and_connect_node(%Node{ + ip: {127, 0, 0, 1}, + port: 3000, + first_public_key: key, + last_public_key: key, + available?: true, + geo_patch: "AAA", + network_patch: "AAA", + authorized?: true, + authorization_date: DateTime.utc_now() + }) + + address = "64F05F5236088FC64D1BB19BD13BC548F1C49A42432AF02AD9024D8A2990B2B4" + b_address = Base.decode16!(address) + + MockClient + |> expect(:send_message, fn _, _, _ -> + {:ok, %FirstAddress{address: b_address}} + end) + + assert %Transaction{data: %TransactionData{content: "yes"}} = + ~s""" + actions triggered_by: transaction do + address = get_genesis_address("64F05F5236088FC64D1BB19BD13BC548F1C49A42432AF02AD9024D8A2990B2B4") + if address == "64F05F5236088FC64D1BB19BD13BC548F1C49A42432AF02AD9024D8A2990B2B4" do + set_content "yes" + else + set_content "no" + end + end + """ + |> Interpreter.sanitize_code() + |> elem(1) + |> ActionInterpreter.parse() + |> elem(2) + |> ActionInterpreter.execute() + end +end diff --git a/test/archethic/contracts/interpreter/condition_test.exs b/test/archethic/contracts/interpreter/condition_test.exs new file mode 100644 index 0000000000..f54c4d540f --- /dev/null +++ b/test/archethic/contracts/interpreter/condition_test.exs @@ -0,0 +1,280 @@ +defmodule Archethic.Contracts.ConditionInterpreterTest do + use ArchethicCase + + alias Archethic.Contracts.ContractConditions, as: Conditions + alias Archethic.Contracts.ConditionInterpreter + alias Archethic.Contracts.Interpreter + + alias Archethic.P2P + alias Archethic.P2P.Node + alias Archethic.P2P.Message.FirstPublicKey + alias Archethic.P2P.Message.FirstAddress + + doctest ConditionInterpreter + + import Mox + + test "should parse map based inherit constraints" do + assert {:ok, :inherit, + %Conditions{ + uco_transfers: + {:==, _, + [ + {:get_in, _, + [ + {:scope, _, nil}, + ["next", "uco_transfers"] + ]}, + {:%{}, [line: 2], + [ + {"7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3", + 1_040_000_000} + ]} + ]} + }} = + """ + condition inherit: [ + uco_transfers: %{ "7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3" => 1040000000 } + ] + """ + |> Interpreter.sanitize_code() + |> elem(1) + |> ConditionInterpreter.parse() + end + + test "should parse multiline inherit constraints" do + assert {:ok, :inherit, + %Conditions{ + uco_transfers: + {:==, _, + [ + {:get_in, _, [{:scope, _, nil}, ["next", "uco_transfers"]]}, + [ + {:%{}, _, + [ + {"to", + "7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3"}, + {"amount", 1_040_000_000} + ]} + ] + ]}, + content: {:==, _, [{:get_in, _, [{:scope, _, nil}, ["next", "content"]]}, "hello"]} + }} = + """ + condition inherit: [ + uco_transfers: [%{ to: "7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3", amount: 1040000000 }], + content: "hello" + ] + + """ + |> Interpreter.sanitize_code() + |> elem(1) + |> ConditionInterpreter.parse() + end + + test "should flatten comparison operators" do + assert {:ok, :inherit, + %Conditions{ + content: + {:>=, [line: 2], + [ + {{:., [line: 2], + [ + {:__aliases__, [alias: Archethic.Contracts.Interpreter.Library], + [:Library]}, + :size + ]}, [line: 2], + [ + {:get_in, [line: 2], [{:scope, [line: 2], nil}, ["next", "content"]]} + ]}, + 10 + ]} + }} = + """ + condition inherit: [ + content: size() >= 10 + ] + """ + |> Interpreter.sanitize_code() + |> elem(1) + |> ConditionInterpreter.parse() + end + + test "should accept conditional code within condition" do + assert {:ok, :inherit, %Conditions{}} = + ~S""" + condition inherit: [ + content: if type == transfer do + regex_match?("hello") + else + regex_match?("hi") + end + ] + + """ + |> Interpreter.sanitize_code() + |> elem(1) + |> ConditionInterpreter.parse() + end + + test "should accept different type of transaction keyword references" do + assert {:ok, :inherit, %Conditions{}} = + ~S""" + condition inherit: [ + content: if next.type == transfer do + "hi" + else + if previous.type == transfer do + "hi" + else + "hello" + end + end + ] + """ + |> Interpreter.sanitize_code() + |> elem(1) + |> ConditionInterpreter.parse() + end + + test "parse invalid uco transfers type definition" do + assert {:error, + "must be a map or a code instruction starting by an comparator - uco_transfers"} = + """ + condition inherit: [ + uco_transfers: [%{ "7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3" => 1040000000 }] + ] + """ + |> Interpreter.sanitize_code() + |> elem(1) + |> ConditionInterpreter.parse() + end + + test "parse invalid token transfers type definition" do + assert {:error, + "must be a map or a code instruction starting by an comparator - token_transfers"} = + """ + condition inherit: [ + token_transfers: [%{ "7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3" => 1040000000 }] + ] + """ + |> Interpreter.sanitize_code() + |> elem(1) + |> ConditionInterpreter.parse() + end + + describe "valid_conditions?" do + test "should validate complex if conditions" do + {:ok, :inherit, conditions} = + ~S""" + condition inherit: [ + type: in?([transfer, token]), + content: if type == transfer do + regex_match?("reason transfer: (.*)") + else + regex_match?("reason token creation: (.*)") + end + ] + """ + |> Interpreter.sanitize_code() + |> elem(1) + |> ConditionInterpreter.parse() + + assert true == + ConditionInterpreter.valid_conditions?(conditions, %{ + "next" => %{"type" => "transfer", "content" => "reason transfer: pay back alice"} + }) + + assert false == + ConditionInterpreter.valid_conditions?(conditions, %{ + "next" => %{"type" => "transfer", "content" => "dummy"} + }) + + assert true == + ConditionInterpreter.valid_conditions?(conditions, %{ + "next" => %{ + "type" => "token", + "content" => "reason token creation: new super token" + } + }) + end + + test "shall get the first address of the chain in the conditions" do + key = <<0::16, :crypto.strong_rand_bytes(32)::binary>> + + P2P.add_and_connect_node(%Node{ + ip: {127, 0, 0, 1}, + port: 3000, + first_public_key: key, + last_public_key: key, + available?: true, + geo_patch: "AAA", + network_patch: "AAA", + authorized?: true, + authorization_date: DateTime.utc_now() + }) + + address = "64F05F5236088FC64D1BB19BD13BC548F1C49A42432AF02AD9024D8A2990B2B4" + b_address = Base.decode16!(address) + + MockClient + |> expect(:send_message, fn _, _, _ -> + {:ok, %FirstAddress{address: b_address}} + end) + + assert true = + ~s""" + condition transaction: [ + address: get_genesis_address() == "64F05F5236088FC64D1BB19BD13BC548F1C49A42432AF02AD9024D8A2990B2B4" + ] + """ + |> Interpreter.sanitize_code() + |> elem(1) + |> ConditionInterpreter.parse() + |> elem(2) + |> ConditionInterpreter.valid_conditions?(%{ + "transaction" => %{"address" => :crypto.strong_rand_bytes(32)} + }) + end + + test "shall get the first public of the chain in the conditions" do + key = <<0::16, :crypto.strong_rand_bytes(32)::binary>> + + P2P.add_and_connect_node(%Node{ + ip: {127, 0, 0, 1}, + port: 3000, + first_public_key: key, + last_public_key: key, + available?: true, + geo_patch: "AAA", + network_patch: "AAA", + authorized?: true, + authorization_date: DateTime.utc_now() + }) + + public_key = "0001DDE54A313E5DCD73E413748CBF6679F07717F8BDC66CBE8F981E1E475A98605C" + b_public_key = Base.decode16!(public_key) + + MockClient + |> expect(:send_message, fn _, _, _ -> + {:ok, %FirstPublicKey{public_key: b_public_key}} + end) + + assert true = + ~s""" + condition transaction: [ + previous_public_key: get_genesis_public_key() == "0001DDE54A313E5DCD73E413748CBF6679F07717F8BDC66CBE8F981E1E475A98605C" + ] + """ + |> Interpreter.sanitize_code() + |> elem(1) + |> ConditionInterpreter.parse() + |> elem(2) + |> ConditionInterpreter.valid_conditions?(%{ + "transaction" => %{ + "previous_public_key" => <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> + } + }) + end + end +end diff --git a/test/archethic/contracts/interpreter_test.exs b/test/archethic/contracts/interpreter_test.exs index 2ce64ec026..1a27cdb996 100644 --- a/test/archethic/contracts/interpreter_test.exs +++ b/test/archethic/contracts/interpreter_test.exs @@ -3,651 +3,75 @@ defmodule Archethic.Contracts.InterpreterTest do use ArchethicCase alias Archethic.Contracts.Contract - alias Archethic.Contracts.Contract.Conditions - alias Archethic.Contracts.Contract.Constants - alias Archethic.Contracts.Contract.Trigger alias Archethic.Contracts.Interpreter - alias Archethic.P2P - alias Archethic.P2P.Node - alias Archethic.P2P.Message.FirstAddress - alias Archethic.P2P.Message.FirstPublicKey alias Archethic.TransactionChain.Transaction alias Archethic.TransactionChain.TransactionData - alias Archethic.TransactionChain.TransactionData.Ledger - alias Archethic.TransactionChain.TransactionData.UCOLedger - alias Archethic.TransactionChain.TransactionData.UCOLedger.Transfer, as: UCOTransfer - - import Mox doctest Interpreter describe "parse/1" do - test "should parse a contract with some standard functions" do - assert {:ok, - %Contract{ - triggers: [ - %Trigger{ - type: :transaction, - actions: { - :__block__, - [], - [ - { - :=, - [line: 2], - [ - {:scope, [line: 2], nil}, - { - :update_in, - [line: 2], - [ - {:scope, [line: 2], nil}, - ["next_transaction"], - {:&, [line: 2], - [ - {{:., [line: 2], - [ - {:__aliases__, - [ - alias: - Archethic.Contracts.Interpreter.TransactionStatements - ], [:TransactionStatements]}, - :set_type - ]}, [line: 2], [{:&, [line: 2], [1]}, "transfer"]} - ]} - ] - } - ] - }, - { - :=, - [line: 3], - [ - {:scope, [line: 3], nil}, - { - :update_in, - [line: 3], - [ - {:scope, [line: 3], nil}, - ["next_transaction"], - { - :&, - [line: 3], - [ - { - {:., [line: 3], - [ - {:__aliases__, - [ - alias: - Archethic.Contracts.Interpreter.TransactionStatements - ], [:TransactionStatements]}, - :add_uco_transfer - ]}, - [line: 3], - [ - {:&, [line: 3], [1]}, - [ - {"to", - <<127, 102, 97, 172, 226, 130, 249, 71, 172, 162, 239, - 148, 125, 1, 189, 220, 144, 198, 95, 9, 238, 130, - 139, 218, 222, 46, 62, 212, 37, 132, 112, 179>>}, - {"amount", 1_040_000_000} - ] - ] - } - ] - } - ] - } - ] - }, - { - :=, - [line: 4], - [ - {:scope, [line: 4], nil}, - { - :update_in, - [line: 4], - [ - {:scope, [line: 4], nil}, - ["next_transaction"], - { - :&, - [line: 4], - [ - { - {:., [line: 4], - [ - {:__aliases__, - [ - alias: - Archethic.Contracts.Interpreter.TransactionStatements - ], [:TransactionStatements]}, - :add_token_transfer - ]}, - [line: 4], - [ - {:&, [line: 4], [1]}, - [ - {"to", - <<48, 103, 4, 85, 113, 62, 44, 190, 207, 148, 89, 18, - 38, 169, 3, 101, 30, 216, 98, 86, 53, 24, 29, 218, - 35, 111, 236, 194, 33, 209, 231, 228>>}, - {"amount", 20_000_000_000}, - {"token_address", - <<174, 180, 166, 245, 171, 109, 130, 190, 34, 60, 88, - 103, 235, 165, 254, 97, 111, 82, 244, 16, 220, 248, - 59, 69, 175, 241, 88, 221, 64, 174, 138, 195>>}, - {"token_id", 0} - ] - ] - } - ] - } - ] - } - ] - }, - { - :=, - [line: 5], - [ - {:scope, [line: 5], nil}, - { - :update_in, - [line: 5], - [ - {:scope, [line: 5], nil}, - ["next_transaction"], - { - :&, - [line: 5], - [ - {{:., [line: 5], - [ - {:__aliases__, - [ - alias: - Archethic.Contracts.Interpreter.TransactionStatements - ], [:TransactionStatements]}, - :set_content - ]}, [line: 5], [{:&, [line: 5], [1]}, "Receipt"]} - ] - } - ] - } - ] - }, - { - :=, - [line: 6], - [ - {:scope, [line: 6], nil}, - { - :update_in, - [line: 6], - [ - {:scope, [line: 6], nil}, - ["next_transaction"], - { - :&, - [line: 6], - [ - { - {:., [line: 6], - [ - {:__aliases__, - [ - alias: - Archethic.Contracts.Interpreter.TransactionStatements - ], [:TransactionStatements]}, - :add_ownership - ]}, - [line: 6], - [ - {:&, [line: 6], [1]}, - [ - {"secret", "MyEncryptedSecret"}, - {"secret_key", "MySecretKey"}, - {"authorized_public_keys", - [ - <<112, 194, 69, 229, 217, 112, 181, 157, 246, 86, 56, - 189, 213, 217, 99, 238, 34, 230, 216, 146, 234, 34, - 77, 136, 9, 208, 251, 117, 208, 177, 144, 122>> - ]} - ] - ] - } - ] - } - ] - } - ] - }, - { - :=, - [line: 7], - [ - {:scope, [line: 7], nil}, - { - :update_in, - [line: 7], - [ - {:scope, [line: 7], nil}, - ["next_transaction"], - { - :&, - [line: 7], - [ - { - {:., [line: 7], - [ - {:__aliases__, - [ - alias: - Archethic.Contracts.Interpreter.TransactionStatements - ], [:TransactionStatements]}, - :add_recipient - ]}, - [line: 7], - [ - {:&, [line: 7], [1]}, - <<120, 39, 60, 92, 188, 235, 134, 23, 245, 67, 128, 204, - 47, 23, 61, 242, 64, 77, 182, 118, 201, 241, 13, 84, - 107, 111, 57, 94, 111, 59, 221, 238>> - ] - } - ] - } - ] - } - ] - } - ] - } - } - ] - }} = + test "should return an error if not conditions or triggers are defined" do + assert {:error, _} = """ - actions triggered_by: transaction do - set_type transfer - add_uco_transfer to: \"7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3\", amount: 1040000000 - add_token_transfer to: \"30670455713E2CBECF94591226A903651ED8625635181DDA236FECC221D1E7E4\", amount: 20000000000, token_address: \"AEB4A6F5AB6D82BE223C5867EBA5FE616F52F410DCF83B45AFF158DD40AE8AC3\", token_id: 0 - set_content \"Receipt\" - add_ownership secret: \"MyEncryptedSecret\", secret_key: \"MySecretKey\", authorized_public_keys: ["70C245E5D970B59DF65638BDD5D963EE22E6D892EA224D8809D0FB75D0B1907A"] - add_recipient \"78273C5CBCEB8617F54380CC2F173DF2404DB676C9F10D546B6F395E6F3BDDEE\" - end + abc """ |> Interpreter.parse() - end - test "should parse a contract with some map based inherit constraints" do - assert {:ok, - %Contract{ - conditions: %{ - inherit: %Conditions{ - uco_transfers: - {:==, _, - [ - {:get_in, _, - [ - {:scope, _, nil}, - ["next", "uco_transfers"] - ]}, - [ - {:%{}, _, - [ - {"to", - <<127, 102, 97, 172, 226, 130, 249, 71, 172, 162, 239, 148, 125, 1, - 189, 220, 144, 198, 95, 9, 238, 130, 139, 218, 222, 46, 62, 212, - 37, 132, 112, 179>>}, - {"amount", 1_040_000_000} - ]} - ] - ]} - } - } - }} = + assert {:error, _} = """ - condition inherit: [ - uco_transfers: [%{ to: "7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3", amount: 1040000000 }] - ] - - actions triggered_by: datetime, at: 1102190390 do - set_type transfer - add_uco_transfer to: \"7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3\", amount: 1040000000 - end + condition """ |> Interpreter.parse() end - test "should parse multiline inherit constraints" do - assert {:ok, - %Contract{ - conditions: %{ - inherit: %Conditions{ - uco_transfers: - {:==, _, - [ - {:get_in, _, [{:scope, _, nil}, ["next", "uco_transfers"]]}, - [ - {:%{}, _, - [ - {"to", - <<127, 102, 97, 172, 226, 130, 249, 71, 172, 162, 239, 148, 125, 1, - 189, 220, 144, 198, 95, 9, 238, 130, 139, 218, 222, 46, 62, 212, - 37, 132, 112, 179>>}, - {"amount", 1_040_000_000} - ]} - ] - ]}, - content: - {:==, _, [{:get_in, _, [{:scope, _, nil}, ["next", "content"]]}, "hello"]} - } - } - }} = - """ - condition inherit: [ - uco_transfers: [%{ to: "7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3", amount: 1040000000 }], - content: "hello" - ] - - """ - |> Interpreter.parse() - end - end - - describe "execute_actions/2" do - test "should evaluate actions based on if statement" do - {:ok, contract} = - ~S""" - actions triggered_by: transaction do - if transaction.previous_public_key == "abc" do - set_content "yes" - else - set_content "no" - end - end - """ - |> Interpreter.parse() - - assert %Transaction{data: %TransactionData{content: "yes"}} = - Interpreter.execute_actions(contract, :transaction, %{ - "transaction" => %{"previous_public_key" => "abc"} - }) - end - - test "should use a variable assignation" do - {:ok, contract} = - """ - actions triggered_by: transaction do - new_content = \"hello\" - set_content new_content - end - """ - |> Interpreter.parse() - - assert %Transaction{data: %TransactionData{content: "hello"}} = - Interpreter.execute_actions(contract, :transaction) - end - - test "should use a interpolation assignation" do - {:ok, contract} = - ~S""" - actions triggered_by: transaction do - new_content = "hello #{2+2}" - set_content new_content - end - """ - |> Interpreter.parse() - - assert %Transaction{data: %TransactionData{content: "hello 4"}} = - Interpreter.execute_actions(contract, :transaction) - end - - test "should flatten comparison operators" do - code = """ - condition inherit: [ - content: size() >= 10 - ] - """ - - {:ok, - %Contract{ - conditions: %{ - inherit: %Conditions{ - content: - {:>=, [line: 2], - [ - {{:., [line: 2], - [ - {:__aliases__, [alias: Archethic.Contracts.Interpreter.Library], - [:Library]}, - :size - ]}, [line: 2], - [ - {:get_in, [line: 2], [{:scope, [line: 2], nil}, ["next", "content"]]} - ]}, - 10 - ]} - } - } - }} = Interpreter.parse(code) - end - - test "should accept conditional code within condition" do - {:ok, %Contract{}} = - ~S""" - condition inherit: [ - content: if type == transfer do - regex_match?("hello") - else - regex_match?("hi") - end - ] - - """ - |> Interpreter.parse() - end - - test "should accept different type of transaction keyword references" do - {:ok, %Contract{}} = - ~S""" - condition inherit: [ - content: if next.type == transfer do - "hi" - else - if previous.type == transfer do - "hi" - else - "hello" - end - end - ] - - condition transaction: [ - content: "#{hash(contract.code)} - YES" - ] - """ - |> Interpreter.parse() - end - end - - describe "execute/2" do - test "should execute complex condition with if statements" do - code = ~S""" - condition inherit: [ - type: in?([transfer, token]), - content: if type == transfer do - regex_match?("reason transfer: (.*)") - else - regex_match?("reason token creation: (.*)") - end, - ] - - condition transaction: [ - content: hash(contract.code) - ] - """ - - {:ok, - %Contract{conditions: %{inherit: inherit_conditions, transaction: transaction_conditions}}} = - Interpreter.parse(code) - - assert true == - Interpreter.valid_conditions?(inherit_conditions, %{ - "next" => %{"type" => "transfer", "content" => "reason transfer: pay back alice"} - }) - - assert false == - Interpreter.valid_conditions?(inherit_conditions, %{ - "next" => %{"type" => "transfer", "content" => "dummy"} - }) - - assert true == - Interpreter.valid_conditions?(inherit_conditions, %{ - "next" => %{ - "type" => "token", - "content" => "reason token creation: new super token" - } - }) - - assert true == - Interpreter.valid_conditions?(transaction_conditions, %{ - "transaction" => %{"content" => :crypto.hash(:sha256, code)}, - "contract" => %{"code" => code} - }) + test "should return an error for unexpected term" do + assert {:error, "unexpected term - @1 - L1"} = "@1" |> Interpreter.parse() end end test "ICO contract parsing" do - {:ok, _} = - """ - condition inherit: [ - type: transfer, - uco_transfers: size() == 1 - # TODO: to provide more security, we should check the destination address is within the previous transaction inputs - ] - - - actions triggered_by: transaction do - # Get the amount of uco send to this contract - amount_send = transaction.uco_transfers[contract.address] - - if amount_send > 0 do - # Convert UCO to the number of tokens to credit. Each UCO worth 10000 token - token_to_credit = amount_send * 10000 - - # Send the new transaction - set_type transfer - add_token_transfer to: transaction.address, token_address: contract.address, amount: token_to_credit, token_id: token_id - end - end - """ - |> Interpreter.parse() + assert {:ok, _} = + """ + condition inherit: [ + token_transfers: size() == 1 + ] + + condition transaction: [ + uco_transfers: size() > 0, + timestamp: transaction.timestamp < 1665750161 + ] + + actions triggered_by: transaction do + # Get the amount of uco send to this contract + amount_send = transaction.uco_transfers[contract.address] + if amount_send > 0 do + # Convert UCO to the number of tokens to credit. Each UCO worth 10 token + token_to_credit = amount_send * 10 + + # Send the new transaction + add_token_transfer to: transaction.address, token_address: contract.address, amount: token_to_credit + end + end + """ + |> Interpreter.parse() end - describe "get_genesis_address/1" do - setup do - key = <<0::16, :crypto.strong_rand_bytes(32)::binary>> - - P2P.add_and_connect_node(%Node{ - ip: {127, 0, 0, 1}, - port: 3000, - first_public_key: key, - last_public_key: key, - available?: true, - geo_patch: "AAA", - network_patch: "AAA", - authorized?: true, - authorization_date: DateTime.utc_now() - }) - - {:ok, [key: key]} - end - - test "shall get the first address of the chain in the conditions" do - address = "64F05F5236088FC64D1BB19BD13BC548F1C49A42432AF02AD9024D8A2990B2B4" - b_address = Base.decode16!(address) - - MockClient - |> expect(:send_message, fn _, _, _ -> - {:ok, %FirstAddress{address: b_address}} - end) - - {:ok, %Contract{conditions: %{transaction: conditions}}} = - ~s""" - condition transaction: [ - address: get_genesis_address() == "64F05F5236088FC64D1BB19BD13BC548F1C49A42432AF02AD9024D8A2990B2B4" - - ] - """ - |> Interpreter.parse() - - assert true = - Interpreter.valid_conditions?( - conditions, - %{"transaction" => %{"address" => :crypto.strong_rand_bytes(32)}} - ) - end - - test "shall get the first public of the chain in the conditions" do - public_key = "0001DDE54A313E5DCD73E413748CBF6679F07717F8BDC66CBE8F981E1E475A98605C" - b_public_key = Base.decode16!(public_key) - - MockClient - |> expect(:send_message, fn _, _, _ -> - {:ok, %FirstPublicKey{public_key: b_public_key}} - end) - - {:ok, %Contract{conditions: %{transaction: conditions}}} = - ~s""" - condition transaction: [ - previous_public_key: get_genesis_public_key() == "0001DDE54A313E5DCD73E413748CBF6679F07717F8BDC66CBE8F981E1E475A98605C" - ] - """ - |> Interpreter.parse() - - assert true = - Interpreter.valid_conditions?( - conditions, - %{ - "transaction" => %{ - "previous_public_key" => - <<0::8, 0::8, :crypto.strong_rand_bytes(32)::binary>> - } - } - ) - end - - test "shall parse get_genesis_address/1 in actions" do - address = "64F05F5236088FC64D1BB19BD13BC548F1C49A42432AF02AD9024D8A2990B2B4" - b_address = Base.decode16!(address) - - MockClient - |> expect(:send_message, fn _, _, _ -> - {:ok, %FirstAddress{address: b_address}} - end) - - {:ok, contract} = - ~s""" - actions triggered_by: transaction do - address = get_genesis_address "64F05F5236088FC64D1BB19BD13BC548F1C49A42432AF02AD9024D8A2990B2B4" - if address == "64F05F5236088FC64D1BB19BD13BC548F1C49A42432AF02AD9024D8A2990B2B4" do - set_content "yes" - else - set_content "no" - end - end - """ - |> Interpreter.parse() - - assert %Transaction{data: %TransactionData{content: "yes"}} = - Interpreter.execute_actions(contract, :transaction) - end + test "schedule transfers parsing" do + assert {:ok, _} = + """ + condition inherit: [ + type: transfer, + uco_transfers: + %{ "0000D574D171A484F8DEAC2D61FC3F7CC984BEB52465D69B3B5F670090742CBF5CC" => 100000000 } + ] + + actions triggered_by: interval, at: "* * * * *" do + set_type transfer + add_uco_transfer to: "0000D574D171A484F8DEAC2D61FC3F7CC984BEB52465D69B3B5F670090742CBF5CC", amount: 100000000 + end + """ + |> Interpreter.parse() end end diff --git a/test/archethic/contracts/loader_test.exs b/test/archethic/contracts/loader_test.exs index 3883db6545..f9e3fefbc3 100644 --- a/test/archethic/contracts/loader_test.exs +++ b/test/archethic/contracts/loader_test.exs @@ -1,10 +1,9 @@ defmodule Archethic.Contracts.LoaderTest do - use ExUnit.Case + use ArchethicCase alias Archethic.ContractRegistry alias Archethic.Contracts.Contract - alias Archethic.Contracts.Contract.Constants - alias Archethic.Contracts.Contract.Trigger + alias Archethic.Contracts.ContractConstants, as: Constants alias Archethic.Contracts.Loader alias Archethic.Contracts.Worker alias Archethic.ContractSupervisor @@ -55,7 +54,7 @@ defmodule Archethic.Contracts.LoaderTest do assert %{ contract: %Contract{ - triggers: [%Trigger{type: :transaction}], + triggers: %{transaction: _}, constants: %Constants{contract: %{"address" => ^contract_address}} } } = :sys.get_state(pid) @@ -164,7 +163,7 @@ defmodule Archethic.Contracts.LoaderTest do assert %{ contract: %Contract{ - triggers: [%Trigger{type: :transaction}], + triggers: %{transaction: _}, constants: %Constants{contract: %{"address" => ^contract_address}} } } = :sys.get_state(pid) diff --git a/test/archethic/contracts/worker_test.exs b/test/archethic/contracts/worker_test.exs index 660e9d2227..459cb462dc 100644 --- a/test/archethic/contracts/worker_test.exs +++ b/test/archethic/contracts/worker_test.exs @@ -4,7 +4,7 @@ defmodule Archethic.Contracts.WorkerTest do alias Archethic.Account alias Archethic.Contracts.Contract - alias Archethic.Contracts.Contract.Constants + alias Archethic.Contracts.ContractConstants, as: Constants alias Archethic.Contracts.Interpreter @@ -141,7 +141,7 @@ defmodule Archethic.Contracts.WorkerTest do } do code = """ condition inherit: [ - uco_transfers: [%{ to: "7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3", amount: 1_040_000_000}] + uco_transfers: %{ "7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3" => 1_040_000_000} ] actions triggered_by: datetime, at: #{DateTime.utc_now() |> DateTime.add(1) |> DateTime.to_unix()} do @@ -175,7 +175,7 @@ defmodule Archethic.Contracts.WorkerTest do } do code = """ condition inherit: [ - uco_transfers: [%{ to: "7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3", amount: 1_040_000_000}] + uco_transfers: %{ "7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3" => 1_040_000_000} ] actions triggered_by: interval, at: "* * * * * *" do @@ -192,9 +192,10 @@ defmodule Archethic.Contracts.WorkerTest do | constants: %Constants{contract: Map.put(constants, "code", code)} } |> Map.update!(:triggers, fn triggers -> - Enum.map(triggers, fn trigger -> - %{trigger | opts: Keyword.put(trigger.opts, :enable_seconds, true)} + Enum.map(triggers, fn {{:interval, interval}, code} -> + {{:interval, {interval, :second}}, code} end) + |> Enum.into(%{}) end) {:ok, _pid} = Worker.start_link(contract) @@ -241,7 +242,7 @@ defmodule Archethic.Contracts.WorkerTest do } do code = """ condition inherit: [ - uco_transfers: [%{ to: "7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3", amount: 1_040_000_000}] + uco_transfers: %{ "7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3" => 1_040_000_000} ] actions triggered_by: transaction do @@ -288,7 +289,7 @@ defmodule Archethic.Contracts.WorkerTest do ] condition inherit: [ - uco_transfers: [%{ to: "7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3", amount: 1_040_000_000}] + uco_transfers: %{ "7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3" => 1_040_000_000} ] actions triggered_by: transaction do @@ -343,7 +344,7 @@ defmodule Archethic.Contracts.WorkerTest do ] condition inherit: [ - uco_transfers: [%{ to: "7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3", amount: 1_040_000_000}] + uco_transfers: %{ "7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3" => 1_040_000_000} ] actions triggered_by: transaction do @@ -355,7 +356,7 @@ defmodule Archethic.Contracts.WorkerTest do ] condition inherit: [ - uco_transfers: [%{ to: \\"7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3\\", amount: 9_200_000_000}] + uco_transfers: %{ \\"7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3\\" => 9_200_000_000} ] actions triggered_by: transaction do @@ -391,7 +392,7 @@ defmodule Archethic.Contracts.WorkerTest do ] condition inherit: [ - uco_transfers: [%{ to: \"7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3\", amount: 9_200_000_000}] + uco_transfers: %{ \"7F6661ACE282F947ACA2EF947D01BDDC90C65F09EE828BDADE2E3ED4258470B3\" => 9_200_000_000} ] actions triggered_by: transaction do diff --git a/test/archethic/contracts_test.exs b/test/archethic/contracts_test.exs index e20df4ff67..6050c5e4fd 100644 --- a/test/archethic/contracts_test.exs +++ b/test/archethic/contracts_test.exs @@ -4,9 +4,8 @@ defmodule Archethic.ContractsTest do alias Archethic.Contracts alias Archethic.Contracts.Contract - alias Archethic.Contracts.Contract.Conditions - alias Archethic.Contracts.Contract.Constants - alias Archethic.Contracts.Contract.Trigger + alias Archethic.Contracts.ContractConditions, as: Conditions + alias Archethic.Contracts.ContractConstants, as: Constants alias Archethic.TransactionChain.Transaction alias Archethic.TransactionChain.TransactionData @@ -222,7 +221,7 @@ defmodule Archethic.ContractsTest do uco_transfers: %{ "3265CCD78CD74984FAB3CC6984D30C8C82044EBBAB1A4FFFB683BDB2D8C5BCF9" => 1000000000} ] - actions triggered_by: interval, at: "0 * * * * *" do + actions triggered_by: interval, at: "0 * * * *" do add_uco_transfer to: \"3265CCD78CD74984FAB3CC6984D30C8C82044EBBAB1A4FFFB683BDB2D8C5BCF9\", amount: 1000000000 end """ @@ -233,8 +232,6 @@ defmodule Archethic.ContractsTest do } } - {:ok, time} = DateTime.new(~D[2016-05-24], ~T[13:26:20], "Etc/UTC") - next_tx = %Transaction{ data: %TransactionData{ code: code, @@ -253,7 +250,8 @@ defmodule Archethic.ContractsTest do } } - assert false == Contracts.accept_new_contract?(previous_tx, next_tx, time) + assert false == + Contracts.accept_new_contract?(previous_tx, next_tx, ~U[2016-05-24 13:26:20Z]) end test "should return false when the transaction have been triggered by interval and the timestamp does match " do @@ -262,7 +260,7 @@ defmodule Archethic.ContractsTest do uco_transfers: %{ "3265CCD78CD74984FAB3CC6984D30C8C82044EBBAB1A4FFFB683BDB2D8C5BCF9" => 1000000000} ] - actions triggered_by: interval, at: "0 * * * * *" do + actions triggered_by: interval, at: "0 * * * *" do add_uco_transfer to: \"3265CCD78CD74984FAB3CC6984D30C8C82044EBBAB1A4FFFB683BDB2D8C5BCF9\", amount: 1000000000 end """ @@ -273,8 +271,6 @@ defmodule Archethic.ContractsTest do } } - {:ok, time} = DateTime.new(~D[2016-05-24], ~T[13:26:00.000999], "Etc/UTC") - next_tx = %Transaction{ data: %TransactionData{ code: code, @@ -293,7 +289,8 @@ defmodule Archethic.ContractsTest do } } - assert true == Contracts.accept_new_contract?(previous_tx, next_tx, time) + assert true == + Contracts.accept_new_contract?(previous_tx, next_tx, ~U[2016-05-24 13:00:00Z]) end test "should return true when the inherit constraint match and when no trigger is specified" do