From bb4f9432b74456414019d043b71c6e3520157500 Mon Sep 17 00:00:00 2001 From: Pawel Nowosielski Date: Sat, 8 Jun 2019 07:02:40 +0200 Subject: [PATCH 01/16] style: move dialyzer ignore directive to dialyzer-igore file --- apps/omg_utils/lib/omg_utils/paginator.ex | 1 - dialyzer.ignore-warnings | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/omg_utils/lib/omg_utils/paginator.ex b/apps/omg_utils/lib/omg_utils/paginator.ex index 4101ff4d81..976932421f 100644 --- a/apps/omg_utils/lib/omg_utils/paginator.ex +++ b/apps/omg_utils/lib/omg_utils/paginator.ex @@ -45,7 +45,6 @@ defmodule OMG.Utils.Paginator do %__MODULE__{data: [], data_paging: data_paging} end - @dialyzer {:nowarn_function, set_data: 2} @spec set_data(list(), t()) :: t() def set_data(data, paginator) when is_list(data), do: %__MODULE__{paginator | data: data} end diff --git a/dialyzer.ignore-warnings b/dialyzer.ignore-warnings index 2166286721..5b2fff9197 100644 --- a/dialyzer.ignore-warnings +++ b/dialyzer.ignore-warnings @@ -248,3 +248,4 @@ Type specification 'Elixir.OMG.Watcher.UtxoSelection':create_advice(#{'Elixir.OM Type specification 'Elixir.OMG.Watcher.UtxoSelection':select_utxo(#{'Elixir.OMG.State.Transaction':currency()=>[#{'__struct__':='Elixir.OMG.Watcher.DB.TxOutput','__meta__':=term(),'amount':=term(),'blknum':=term(),'creating_deposit':=term(),'creating_transaction':=term(),'creating_txhash':=term(),'currency':=term(),'deposit':=term(),'exit':=term(),'oindex':=term(),'owner':=term(),'proof':=term(),'spending_exit':=term(),'spending_transaction':=term(),'spending_tx_oindex':=term(),'spending_txhash':=term(),'txindex':=term()}]},#{'Elixir.OMG.State.Transaction':currency()=>pos_integer()}) -> [{'Elixir.OMG.State.Transaction':currency(),{integer(),[#{'__struct__':='Elixir.OMG.Watcher.DB.TxOutput','__meta__':=term(),'amount':=term(),'blknum':=term(),'creating_deposit':=term(),'creating_transaction':=term(),'creating_txhash':=term(),'currency':=term(),'deposit':=term(),'exit':=term(),'oindex':=term(),'owner':=term(),'proof':=term(),'spending_exit':=term(),'spen Type specification 'Elixir.OMG.WatcherRPC.Web.Validator.Constraints':parse(#{binary()=>any()}) -> {'ok','Elixir.Keyword':t()} | {'error',any()} is a subtype of the success typing: 'Elixir.OMG.WatcherRPC.Web.Validator.Constraints':parse(_) -> any() Type specification 'Elixir.OMG.WatcherRPC.Web.Validator.Order':parse(map()) -> {'ok','Elixir.OMG.Watcher.UtxoSelection':order_t()} | {'error',any()} is a subtype of the success typing: 'Elixir.OMG.WatcherRPC.Web.Validator.Order':parse(map()) -> {'error',_} | {'ok',#{'fee':=#{'amount':=_, 'currency':=_}, 'metadata':=_, 'owner':=_, 'payments':=[any()]}} +Type specification 'Elixir.OMG.Utils.Paginator':set_data([any()],t()) -> t() is a subtype of the success typing: 'Elixir.OMG.Utils.Paginator':set_data(maybe_improper_list(),#{'__struct__':='Elixir.OMG.Utils.Paginator', 'data':=_, _=>_}) -> #{'__struct__':='Elixir.OMG.Utils.Paginator', 'data':=maybe_improper_list(), _=>_} From 3c8a2a21c75e7d04334d8776646047cc7e2971a7 Mon Sep 17 00:00:00 2001 From: Ino Murko Date: Sat, 8 Jun 2019 09:58:55 +0200 Subject: [PATCH 02/16] fix: get cors settings from proper app --- apps/omg_watcher_rpc/lib/web/endpoint.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/omg_watcher_rpc/lib/web/endpoint.ex b/apps/omg_watcher_rpc/lib/web/endpoint.ex index 25780a6ed8..11fa31b3b3 100644 --- a/apps/omg_watcher_rpc/lib/web/endpoint.ex +++ b/apps/omg_watcher_rpc/lib/web/endpoint.ex @@ -32,7 +32,7 @@ defmodule OMG.WatcherRPC.Web.Endpoint do plug(Plug.MethodOverride) plug(Plug.Head) - if Application.get_env(:omg_watcher, OMG.WatcherRPC.Web.Endpoint)[:enable_cors], + if Application.get_env(:omg_watcher_rpc, OMG.WatcherRPC.Web.Endpoint)[:enable_cors], do: plug(CORSPlug) plug(OMG.WatcherRPC.Web.Router) From 14736f5b9fe221a8d137c0830263e5300cba442e Mon Sep 17 00:00:00 2001 From: Ino Murko Date: Thu, 6 Jun 2019 15:04:24 +0200 Subject: [PATCH 03/16] fix: general fixes --- apps/omg/lib/omg/state/core.ex | 27 +++++++++++-------- apps/omg/lib/omg/utxo.ex | 2 +- apps/omg_db/lib/db.ex | 4 +++ apps/omg_db/lib/omg_db/leveldb/core.ex | 5 ++++ apps/omg_db/lib/omg_db/rocksdb/core.ex | 7 +++++ .../lib/omg_watcher/api/account.ex | 1 + .../lib/omg_watcher/block_getter.ex | 15 ++++++----- docs/transaction_validation.md | 4 +-- mix.exs | 5 ++-- mix.lock | 4 +-- 10 files changed, 49 insertions(+), 25 deletions(-) diff --git a/apps/omg/lib/omg/state/core.ex b/apps/omg/lib/omg/state/core.ex index 4950740026..2030461677 100644 --- a/apps/omg/lib/omg/state/core.ex +++ b/apps/omg/lib/omg/state/core.ex @@ -81,20 +81,20 @@ defmodule OMG.State.Core do } @type db_update :: - {:put, :utxo, {Utxo.Position.db_t(), map}} + {:put, :utxo, {Utxo.Position.db_t(), map()}} | {:delete, :utxo, Utxo.Position.db_t()} - | {:put, :child_top_block_number, pos_integer} - | {:put, :last_deposit_child_blknum, pos_integer} + | {:put, :child_top_block_number, pos_integer()} + | {:put, :last_deposit_child_blknum, pos_integer()} | {:put, :block, Block.db_t()} @type exitable_utxos :: %{ creating_txhash: Transaction.tx_hash(), owner: Crypto.address_t(), currency: Crypto.address_t(), - amount: non_neg_integer, - blknum: pos_integer, - txindex: non_neg_integer, - oindex: non_neg_integer + amount: non_neg_integer(), + blknum: pos_integer(), + txindex: non_neg_integer(), + oindex: non_neg_integer() } @doc """ @@ -102,9 +102,9 @@ defmodule OMG.State.Core do """ @spec extract_initial_state( utxos_query_result :: [list({OMG.DB.utxo_pos_db_t(), OMG.Utxo.t()})], - height_query_result :: non_neg_integer | :not_found, - last_deposit_child_blknum_query_result :: non_neg_integer | :not_found, - child_block_interval :: pos_integer + height_query_result :: non_neg_integer() | :not_found, + last_deposit_child_blknum_query_result :: non_neg_integer() | :not_found, + child_block_interval :: pos_integer() ) :: {:ok, t()} | {:error, :last_deposit_not_found | :top_block_number_not_found} def extract_initial_state( utxos_query_result, @@ -204,15 +204,20 @@ defmodule OMG.State.Core do defp get_input_utxos(utxos, inputs) do inputs |> Enum.reduce_while({:ok, []}, fn input, acc -> get_utxos(utxos, input, acc) end) + |> reverse() end defp get_utxos(utxos, position, {:ok, acc}) do case Map.get(utxos, position) do nil -> {:halt, {:error, :utxo_not_found}} - found -> {:cont, {:ok, acc ++ [found]}} + found -> {:cont, {:ok, [found | acc]}} end end + @spec reverse({:ok, any()} | {:error, :utxo_not_found}) :: {:ok, list(any())} | {:error, :utxo_not_found} + defp reverse({:ok, input_utxos}), do: {:ok, Enum.reverse(input_utxos)} + defp reverse({:error, :utxo_not_found} = result), do: result + defp get_amounts_by_currency(utxos) do utxos |> Enum.group_by(fn %{currency: currency} -> currency end, fn %{amount: amount} -> amount end) diff --git a/apps/omg/lib/omg/utxo.ex b/apps/omg/lib/omg/utxo.ex index dc0d49adcd..de7a625aaf 100644 --- a/apps/omg/lib/omg/utxo.ex +++ b/apps/omg/lib/omg/utxo.ex @@ -26,7 +26,7 @@ defmodule OMG.Utxo do creating_txhash: Transaction.tx_hash(), owner: Crypto.address_t(), currency: Crypto.address_t(), - amount: non_neg_integer + amount: non_neg_integer() } @doc """ diff --git a/apps/omg_db/lib/db.ex b/apps/omg_db/lib/db.ex index 28fa6f7757..d21b93f776 100644 --- a/apps/omg_db/lib/db.ex +++ b/apps/omg_db/lib/db.ex @@ -139,9 +139,13 @@ defmodule OMG.DB do """ def single_value_parameter_names do [ + # child chain - used at block forming :child_top_block_number, + # watcher and child chain :last_deposit_child_blknum, + # watcher :last_block_getter_eth_height, + # watcher and child chain :last_depositor_eth_height, :last_convenience_deposit_processor_eth_height, :last_exiter_eth_height, diff --git a/apps/omg_db/lib/omg_db/leveldb/core.ex b/apps/omg_db/lib/omg_db/leveldb/core.ex index 8bf4f6582b..d0b397b1a1 100644 --- a/apps/omg_db/lib/omg_db/leveldb/core.ex +++ b/apps/omg_db/lib/omg_db/leveldb/core.ex @@ -24,10 +24,15 @@ defmodule OMG.DB.LevelDB.Core do @keys_prefixes %{ block: "b", block_hash: "bn", + # watcher and child chain utxo: "u", + # watcher and child chain exit_info: "e", + # watcher only in_flight_exit_info: "ife", + # watcher only competitor_info: "ci", + # watcher only spend: "s" } diff --git a/apps/omg_db/lib/omg_db/rocksdb/core.ex b/apps/omg_db/lib/omg_db/rocksdb/core.ex index 8493904bc8..00f568e0a2 100644 --- a/apps/omg_db/lib/omg_db/rocksdb/core.ex +++ b/apps/omg_db/lib/omg_db/rocksdb/core.ex @@ -25,12 +25,19 @@ defmodule OMG.DB.RocksDB.Core do # prefix extractor to reduce the number of IO scans # more https://github.com/facebook/rocksdb/wiki/Prefix-Seek-API-Changes @keys_prefixes %{ + # watcher (Exit Processor) and child chain (Fresh Blocks) block: "block", + # watcher (Exit Processor) and child chain (Block Queue) block_hash: "hashb", + # watcher and child chain utxo: "utxoi", + # watcher and child chain exit_info: "exiti", + # watcher only in_flight_exit_info: "infle", + # watcher only competitor_info: "compi", + # watcher only spend: "spend" } diff --git a/apps/omg_watcher/lib/omg_watcher/api/account.ex b/apps/omg_watcher/lib/omg_watcher/api/account.ex index 2656722e81..2d6e9ff140 100644 --- a/apps/omg_watcher/lib/omg_watcher/api/account.ex +++ b/apps/omg_watcher/lib/omg_watcher/api/account.ex @@ -38,6 +38,7 @@ defmodule OMG.Watcher.API.Account do @doc """ Gets all utxos belonging to the given address. Slow operation, compatible with security-critical. + TODO: what's the actuall problem here? Storage model? """ @spec get_exitable_utxos(OMG.Crypto.address_t()) :: list(OMG.State.Core.exitable_utxos()) def get_exitable_utxos(address) do diff --git a/apps/omg_watcher/lib/omg_watcher/block_getter.ex b/apps/omg_watcher/lib/omg_watcher/block_getter.ex index 90c2acdaf8..f9cacca40a 100644 --- a/apps/omg_watcher/lib/omg_watcher/block_getter.ex +++ b/apps/omg_watcher/lib/omg_watcher/block_getter.ex @@ -119,17 +119,17 @@ defmodule OMG.Watcher.BlockGetter do ) do with {:ok, _} <- Core.chain_ok(state), tx_exec_results = for(tx <- transactions, do: OMG.State.exec(tx, :ignore)), - {:ok, state} <- Core.validate_executions(tx_exec_results, to_apply, state) do + {:ok, state2} <- Core.validate_executions(tx_exec_results, to_apply, state) do _ = to_apply - |> Core.ensure_block_imported_once(state) + |> Core.ensure_block_imported_once(state2) |> Enum.each(&DB.Transaction.update_with/1) - state = run_block_download_task(state) + state3 = run_block_download_task(state2) {:ok, db_updates_from_state} = OMG.State.close_block(eth_height) - {state, synced_height, db_updates} = Core.apply_block(state, to_apply) + {state4, synced_height, db_updates} = Core.apply_block(state3, to_apply) _ = Logger.debug("Synced height update: #{inspect(db_updates)}") @@ -137,9 +137,9 @@ defmodule OMG.Watcher.BlockGetter do :ok = check_in_to_coordinator(synced_height) exit_processor_results = ExitProcessor.check_validity() - state = Core.consider_exits(state, exit_processor_results) + state5 = Core.consider_exits(state4, exit_processor_results) - :ok = update_status(state) + :ok = update_status(state5) _ = Logger.info( @@ -147,7 +147,7 @@ defmodule OMG.Watcher.BlockGetter do "with #{inspect(length(transactions))} txs" ) - {:noreply, state} + {:noreply, state5} else {{:error, _} = error, new_state} -> :ok = update_status(new_state) @@ -156,6 +156,7 @@ defmodule OMG.Watcher.BlockGetter do {:error, _} = error -> :ok = update_status(state) + # TODO Alarm _ = Logger.warn("Chain already invalid before applying block #{inspect(blknum)} because of #{inspect(error)}") {:noreply, state} end diff --git a/docs/transaction_validation.md b/docs/transaction_validation.md index 625087901b..fde5e09c0a 100644 --- a/docs/transaction_validation.md +++ b/docs/transaction_validation.md @@ -4,7 +4,7 @@ NOTE: * input = utxo This document presents current way of stateless and stateful validation of -`OMG.ChildChain.submit(encoded_signed_tx)` method. +`OMG.ChildChain.submit(encoded_signed_tx)` function. #### Stateless validation @@ -25,7 +25,7 @@ This document presents current way of stateless and stateful validation of 1. Validating block size * if the number of transactions in block exceeds limit then `{:error, :too_many_transactions_in_block}` -2. Checking correction of input positions +2. Checking correctness of input positions * if the input is from the future block then `{:error, :input_utxo_ahead_of_state}` * if the input does not exists then `{:error, :utxo_not_found}` * if the owner of input does not match with spender then `{:error, :unauthorized_spent}` diff --git a/mix.exs b/mix.exs index a680619e4c..3007077bb9 100644 --- a/mix.exs +++ b/mix.exs @@ -27,14 +27,14 @@ defmodule OMG.Umbrella.MixProject do [ {:distillery, "~> 2.0", runtime: false}, {:dialyxir, "~> 1.0.0-rc.6", only: [:dev, :test], runtime: false}, - {:credo, "~> 1.0.0", only: [:dev, :test], runtime: false}, + {:credo, "~> 1.0.5", only: [:dev, :test], runtime: false}, {:excoveralls, "~> 0.11.1", only: [:test], runtime: false}, {:licensir, "~> 0.2.0", only: :dev, runtime: false}, { :ex_unit_fixtures, git: "https://github.com/omisego/ex_unit_fixtures.git", branch: "feature/require_files_not_load", only: [:test] }, - {:ex_doc, "~> 0.19", only: :dev, runtime: false}, + {:ex_doc, "~> 0.20.2", only: :dev, runtime: false}, {:appsignal, "~> 1.9"}, {:libsecp256k1, git: "https://github.com/InoMurko/libsecp256k1.git", @@ -59,6 +59,7 @@ defmodule OMG.Umbrella.MixProject do [ flags: [:specdiffs, :error_handling, :race_conditions, :underspecs, :unknown, :unmatched_returns], ignore_warnings: "dialyzer.ignore-warnings", + list_unused_filters: true, plt_add_apps: plt_apps() ] end diff --git a/mix.lock b/mix.lock index 3a1b837d76..244d4505d5 100644 --- a/mix.lock +++ b/mix.lock @@ -9,7 +9,7 @@ "cors_plug": {:hex, :cors_plug, "2.0.0", "238ddb479f92b38f6dc1ae44b8d81f0387f9519101a6da442d543ab70ee0e482", [:mix], [{:plug, "~> 1.3 or ~> 1.4 or ~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "cowboy": {:hex, :cowboy, "1.1.2", "61ac29ea970389a88eca5a65601460162d370a70018afe6f949a29dca91f3bb0", [:rebar3], [{:cowlib, "~> 1.0.2", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3.2", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"}, "cowlib": {:hex, :cowlib, "1.0.2", "9d769a1d062c9c3ac753096f868ca121e2730b9a377de23dec0f7e08b1df84ee", [:make], [], "hexpm"}, - "credo": {:hex, :credo, "1.0.4", "d2214d4cc88c07f54004ffd5a2a27408208841be5eca9f5a72ce9e8e835f7ede", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, + "credo": {:hex, :credo, "1.0.5", "fdea745579f8845315fe6a3b43e2f9f8866839cfbc8562bb72778e9fdaa94214", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, "db_connection": {:hex, :db_connection, "2.0.6", "bde2f85d047969c5b5800cb8f4b3ed6316c8cb11487afedac4aa5f93fd39abfa", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, "decimal": {:hex, :decimal, "1.7.0", "30d6b52c88541f9a66637359ddf85016df9eb266170d53105f02e4a67e00c5aa", [:mix], [], "hexpm"}, "decorator": {:hex, :decorator, "1.2.4", "31dfff6143d37f0b68d0bffb3b9f18ace14fea54d4f1b5e4f86ead6f00d9ff6e", [:mix], [], "hexpm"}, @@ -24,7 +24,7 @@ "erlexec": {:hex, :erlexec, "1.9.5", "5ac0f70fb43c298a60b65a3e1bcad58714b0b1f6cd0a70b1e116ff42e65f954c", [:rebar3], [], "hexpm"}, "ethereumex": {:hex, :ethereumex, "0.5.4", "f4dc7f8dd6b141d2fd28c7c98e54c361ef2e7cc707b3147a216bf4857077033b", [:mix], [{:httpoison, "~> 1.4.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5.1", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm"}, "ex_abi": {:hex, :ex_abi, "0.2.1", "e8224ef22782b58d535bf3e29e73379af48df52cc9a941746980d14b32e793c2", [:mix], [{:exth_crypto, "~> 0.1.6", [hex: :exth_crypto, repo: "hexpm", optional: false]}], "hexpm"}, - "ex_doc": {:hex, :ex_doc, "0.19.3", "3c7b0f02851f5fc13b040e8e925051452e41248f685e40250d7e40b07b9f8c10", [:mix], [{:earmark, "~> 1.2", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, + "ex_doc": {:hex, :ex_doc, "0.20.2", "1bd0dfb0304bade58beb77f20f21ee3558cc3c753743ae0ddbb0fd7ba2912331", [:mix], [{:earmark, "~> 1.3", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, "ex_rlp": {:hex, :ex_rlp, "0.5.2", "7f4ce7bd55e543c054ce6d49629b01e9833c3462e3d547952be89865f39f2c58", [:mix], [], "hexpm"}, "ex_unit_fixtures": {:git, "https://github.com/omisego/ex_unit_fixtures.git", "4a099c621dc70e0d65cb9619b38192e31ec5f504", [branch: "feature/require_files_not_load"]}, "excoveralls": {:hex, :excoveralls, "0.11.1", "dd677fbdd49114fdbdbf445540ec735808250d56b011077798316505064edb2c", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, From 68844e84296d5df5af9f04ded583a5292835e4bf Mon Sep 17 00:00:00 2001 From: Ino Murko Date: Thu, 6 Jun 2019 17:38:50 +0200 Subject: [PATCH 04/16] fix: state refactor --- .../lib/omg_watcher/block_getter.ex | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/apps/omg_watcher/lib/omg_watcher/block_getter.ex b/apps/omg_watcher/lib/omg_watcher/block_getter.ex index f9cacca40a..85d82aa631 100644 --- a/apps/omg_watcher/lib/omg_watcher/block_getter.ex +++ b/apps/omg_watcher/lib/omg_watcher/block_getter.ex @@ -119,27 +119,26 @@ defmodule OMG.Watcher.BlockGetter do ) do with {:ok, _} <- Core.chain_ok(state), tx_exec_results = for(tx <- transactions, do: OMG.State.exec(tx, :ignore)), - {:ok, state2} <- Core.validate_executions(tx_exec_results, to_apply, state) do + {:ok, state} <- Core.validate_executions(tx_exec_results, to_apply, state) do _ = to_apply - |> Core.ensure_block_imported_once(state2) + |> Core.ensure_block_imported_once(state) |> Enum.each(&DB.Transaction.update_with/1) - state3 = run_block_download_task(state2) - - {:ok, db_updates_from_state} = OMG.State.close_block(eth_height) + {state, synced_height, db_updates} = + state + |> run_block_download_task() + |> Core.apply_block(to_apply) - {state4, synced_height, db_updates} = Core.apply_block(state3, to_apply) + exit_processor_results = ExitProcessor.check_validity() + state = Core.consider_exits(state, exit_processor_results) _ = Logger.debug("Synced height update: #{inspect(db_updates)}") + {:ok, db_updates_from_state} = OMG.State.close_block(eth_height) :ok = OMG.DB.multi_update(db_updates ++ db_updates_from_state) :ok = check_in_to_coordinator(synced_height) - - exit_processor_results = ExitProcessor.check_validity() - state5 = Core.consider_exits(state4, exit_processor_results) - - :ok = update_status(state5) + :ok = update_status(state) _ = Logger.info( @@ -147,7 +146,7 @@ defmodule OMG.Watcher.BlockGetter do "with #{inspect(length(transactions))} txs" ) - {:noreply, state5} + {:noreply, state} else {{:error, _} = error, new_state} -> :ok = update_status(new_state) @@ -156,7 +155,6 @@ defmodule OMG.Watcher.BlockGetter do {:error, _} = error -> :ok = update_status(state) - # TODO Alarm _ = Logger.warn("Chain already invalid before applying block #{inspect(blknum)} because of #{inspect(error)}") {:noreply, state} end From fabe5d669c80609e1ec6dd75a5f89c5bd1e70ec5 Mon Sep 17 00:00:00 2001 From: Ino Murko Date: Thu, 6 Jun 2019 17:50:59 +0200 Subject: [PATCH 05/16] fix: further state refactor --- apps/omg_watcher/lib/omg_watcher/block_getter.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/omg_watcher/lib/omg_watcher/block_getter.ex b/apps/omg_watcher/lib/omg_watcher/block_getter.ex index 85d82aa631..0968d38473 100644 --- a/apps/omg_watcher/lib/omg_watcher/block_getter.ex +++ b/apps/omg_watcher/lib/omg_watcher/block_getter.ex @@ -125,14 +125,14 @@ defmodule OMG.Watcher.BlockGetter do |> Core.ensure_block_imported_once(state) |> Enum.each(&DB.Transaction.update_with/1) + exit_processor_results = ExitProcessor.check_validity() + {state, synced_height, db_updates} = state |> run_block_download_task() + |> Core.consider_exits(exit_processor_results) |> Core.apply_block(to_apply) - exit_processor_results = ExitProcessor.check_validity() - state = Core.consider_exits(state, exit_processor_results) - _ = Logger.debug("Synced height update: #{inspect(db_updates)}") {:ok, db_updates_from_state} = OMG.State.close_block(eth_height) From 39768370bf5d22ea07ca2837169a710f06401d03 Mon Sep 17 00:00:00 2001 From: Ino Murko Date: Thu, 6 Jun 2019 18:54:27 +0200 Subject: [PATCH 06/16] refactor: splitting the transaction execution into smaller bits --- apps/omg/lib/omg/state/core.ex | 231 ++++++------------ .../lib/omg/state/transaction/validator.ex | 91 +++++++ 2 files changed, 171 insertions(+), 151 deletions(-) create mode 100644 apps/omg/lib/omg/state/transaction/validator.ex diff --git a/apps/omg/lib/omg/state/core.ex b/apps/omg/lib/omg/state/core.ex index 2030461677..ed82db9b90 100644 --- a/apps/omg/lib/omg/state/core.ex +++ b/apps/omg/lib/omg/state/core.ex @@ -18,8 +18,6 @@ defmodule OMG.State.Core do All spend transactions, deposits and exits should sync on this for validity of moving funds. """ - @maximum_block_size 65_536 - defstruct [:height, :last_deposit_child_blknum, :utxos, pending_txs: [], tx_index: 0] alias OMG.Block @@ -27,6 +25,7 @@ defmodule OMG.State.Core do alias OMG.Fees alias OMG.State.Core alias OMG.State.Transaction + alias OMG.State.Transaction.Validator alias OMG.Utxo use OMG.Utils.LoggerExt @@ -118,8 +117,7 @@ defmodule OMG.State.Core do height = height_query_result + child_block_interval utxos = - utxos_query_result - |> Enum.into(%{}, fn {db_position, db_utxo} -> + Enum.into(utxos_query_result, %{}, fn {db_position, db_utxo} -> {Utxo.Position.from_db_key(db_position), Utxo.from_db_value(db_utxo)} end) @@ -168,11 +166,11 @@ defmodule OMG.State.Core do tx_hash = Transaction.raw_txhash(tx) outputs = Transaction.get_outputs(tx) - with :ok <- validate_block_size(state), - {:ok, input_amounts_by_currency} <- correct_inputs?(state, tx), - output_amounts_by_currency = get_amounts_by_currency(outputs), - :ok <- amounts_add_up?(input_amounts_by_currency, output_amounts_by_currency), - :ok <- transaction_covers_fee?(input_amounts_by_currency, output_amounts_by_currency, fees) do + with :ok <- Validator.validate_block_size(state), + {:ok, input_amounts_by_currency} <- Validator.correct_inputs?(state, tx), + output_amounts_by_currency = Validator.get_amounts_by_currency(outputs), + :ok <- Validator.amounts_add_up?(input_amounts_by_currency, output_amounts_by_currency), + :ok <- Validator.transaction_covers_fee?(input_amounts_by_currency, output_amounts_by_currency, fees) do {:ok, {tx_hash, height, tx_index}, state |> apply_spend(tx) @@ -182,98 +180,19 @@ defmodule OMG.State.Core do end end - defp correct_inputs?(%Core{utxos: utxos} = state, tx) do - inputs = Transaction.get_inputs(tx) - - with :ok <- inputs_not_from_future_block?(state, inputs), - {:ok, input_utxos} <- get_input_utxos(utxos, inputs), - input_utxos_owners <- Enum.map(input_utxos, fn %{owner: owner} -> owner end), - :ok <- Transaction.Recovered.all_spenders_authorized(tx, input_utxos_owners) do - {:ok, get_amounts_by_currency(input_utxos)} - end - end - - defp inputs_not_from_future_block?(%__MODULE__{height: blknum}, inputs) do - no_utxo_from_future_block = - inputs - |> Enum.all?(fn Utxo.position(input_blknum, _, _) -> blknum >= input_blknum end) - - if no_utxo_from_future_block, do: :ok, else: {:error, :input_utxo_ahead_of_state} - end - - defp get_input_utxos(utxos, inputs) do - inputs - |> Enum.reduce_while({:ok, []}, fn input, acc -> get_utxos(utxos, input, acc) end) - |> reverse() - end - - defp get_utxos(utxos, position, {:ok, acc}) do - case Map.get(utxos, position) do - nil -> {:halt, {:error, :utxo_not_found}} - found -> {:cont, {:ok, [found | acc]}} - end - end - - @spec reverse({:ok, any()} | {:error, :utxo_not_found}) :: {:ok, list(any())} | {:error, :utxo_not_found} - defp reverse({:ok, input_utxos}), do: {:ok, Enum.reverse(input_utxos)} - defp reverse({:error, :utxo_not_found} = result), do: result - - defp get_amounts_by_currency(utxos) do - utxos - |> Enum.group_by(fn %{currency: currency} -> currency end, fn %{amount: amount} -> amount end) - |> Enum.map(fn {currency, amounts} -> {currency, Enum.sum(amounts)} end) - |> Map.new() - end - - defp amounts_add_up?(input_amounts, output_amounts) do - for {output_currency, output_amount} <- Map.to_list(output_amounts) do - input_amount = Map.get(input_amounts, output_currency, 0) - input_amount >= output_amount - end - |> Enum.all?() - |> if(do: :ok, else: {:error, :amounts_do_not_add_up}) - end - - def transaction_covers_fee?(input_amounts, output_amounts, fees) do - Fees.covered?(input_amounts, output_amounts, fees) - |> if(do: :ok, else: {:error, :fees_not_covered}) - end - - defp add_pending_tx(%Core{pending_txs: pending_txs, tx_index: tx_index} = state, %Transaction.Recovered{} = new_tx) do - %Core{ - state - | tx_index: tx_index + 1, - pending_txs: [new_tx | pending_txs] - } - end - - defp apply_spend(%Core{height: height, tx_index: tx_index, utxos: utxos} = state, tx) do - new_utxos_map = tx |> non_zero_utxos_from(height, tx_index) |> Map.new() - - inputs = Transaction.get_inputs(tx) - utxos = Map.drop(utxos, inputs) - %Core{state | utxos: Map.merge(utxos, new_utxos_map)} - end - - defp non_zero_utxos_from(tx, height, tx_index) do - tx - |> utxos_from(height, tx_index) - |> Enum.filter(fn {_key, value} -> is_non_zero_amount?(value) end) - end - - defp utxos_from(tx, height, tx_index) do - hash = Transaction.raw_txhash(tx) - outputs = Transaction.get_outputs(tx) - - for {%{owner: owner, currency: currency, amount: amount}, oindex} <- Enum.with_index(outputs) do - {Utxo.position(height, tx_index, oindex), - %Utxo{owner: owner, currency: currency, amount: amount, creating_txhash: hash}} - end + @doc """ + Filter user utxos from db response. + It may take a while for a large response from db + """ + @spec standard_exitable_utxos(list({OMG.DB.utxo_pos_db_t(), OMG.Utxo.t()}), Crypto.address_t()) :: + list(exitable_utxos) + def standard_exitable_utxos(utxos_query_result, address) do + Stream.filter(utxos_query_result, fn {_, %{owner: owner}} -> owner == address end) + |> Enum.map(fn {{blknum, txindex, oindex}, utxo} -> + utxo |> Map.put(:blknum, blknum) |> Map.put(:txindex, txindex) |> Map.put(:oindex, oindex) + end) end - defp is_non_zero_amount?(%{amount: 0}), do: false - defp is_non_zero_amount?(%{amount: _}), do: true - @doc """ - Generates block and calculates it's root hash for submission - generates triggers for events @@ -343,38 +262,6 @@ defmodule OMG.State.Core do {:ok, {event_triggers, db_updates}, new_state} end - defp utxo_to_db_put({utxo_pos, utxo}), - do: {:put, :utxo, {Utxo.Position.to_db_key(utxo_pos), Utxo.to_db_value(utxo)}} - - defp deposit_to_utxo(%{blknum: blknum, currency: cur, owner: owner, amount: amount}) do - {Utxo.position(blknum, 0, 0), %Utxo{amount: amount, currency: cur, owner: owner}} - end - - defp get_last_deposit_child_blknum(deposits, current_height) do - if Enum.empty?(deposits) do - current_height - else - deposits - |> Enum.max_by(& &1.blknum) - |> Map.get(:blknum) - end - end - - defp last_deposit_child_blknum_db_update(deposits, last_deposit_child_blknum) do - if Enum.empty?(deposits) do - [] - else - [{:put, :last_deposit_child_blknum, last_deposit_child_blknum}] - end - end - - defp validate_block_size(%__MODULE__{tx_index: number_of_transactions_in_block}) do - case number_of_transactions_in_block == @maximum_block_size do - true -> {:error, :too_many_transactions_in_block} - false -> :ok - end - end - @doc """ Spends exited utxos. Accepts both a list of utxo positions (decoded) or full exit info from an event. @@ -432,15 +319,6 @@ defmodule OMG.State.Core do {:ok, {db_updates, validities}, new_state} end - defp find_utxo_matching_piggyback(%{tx_hash: tx_hash, output_index: oindex} = piggyback, utxos) do - # oindex in contract is 0-7 where 4-7 are outputs - oindex = oindex - 4 - - position = Enum.find(utxos, &match?({Utxo.position(_, _, ^oindex), %Utxo{creating_txhash: ^tx_hash}}, &1)) - - {piggyback, position} - end - @doc """ Checks if utxo exists """ @@ -479,16 +357,67 @@ defmodule OMG.State.Core do Enum.concat(db_updates_new_utxos, db_updates_spent_utxos) end - @doc """ - Filter user utxos from db response. - It may take a while for a large response from db - """ - @spec standard_exitable_utxos(list({OMG.DB.utxo_pos_db_t(), OMG.Utxo.t()}), Crypto.address_t()) :: - list(exitable_utxos) - def standard_exitable_utxos(utxos_query_result, address) do - Stream.filter(utxos_query_result, fn {_, %{owner: owner}} -> owner == address end) - |> Enum.map(fn {{blknum, txindex, oindex}, utxo} -> - utxo |> Map.put(:blknum, blknum) |> Map.put(:txindex, txindex) |> Map.put(:oindex, oindex) - end) + defp add_pending_tx(%Core{pending_txs: pending_txs, tx_index: tx_index} = state, %Transaction.Recovered{} = new_tx) do + %Core{ + state + | tx_index: tx_index + 1, + pending_txs: [new_tx | pending_txs] + } + end + + defp apply_spend(%Core{height: height, tx_index: tx_index, utxos: utxos} = state, tx) do + new_utxos_map = tx |> non_zero_utxos_from(height, tx_index) |> Map.new() + + inputs = Transaction.get_inputs(tx) + utxos = Map.drop(utxos, inputs) + %Core{state | utxos: Map.merge(utxos, new_utxos_map)} + end + + defp non_zero_utxos_from(tx, height, tx_index) do + tx + |> utxos_from(height, tx_index) + |> Enum.filter(fn {_key, value} -> is_non_zero_amount?(value) end) + end + + defp utxos_from(tx, height, tx_index) do + hash = Transaction.raw_txhash(tx) + outputs = Transaction.get_outputs(tx) + + for {%{owner: owner, currency: currency, amount: amount}, oindex} <- Enum.with_index(outputs) do + {Utxo.position(height, tx_index, oindex), + %Utxo{owner: owner, currency: currency, amount: amount, creating_txhash: hash}} + end + end + + defp is_non_zero_amount?(%{amount: 0}), do: false + defp is_non_zero_amount?(%{amount: _}), do: true + + defp utxo_to_db_put({utxo_pos, utxo}), + do: {:put, :utxo, {Utxo.Position.to_db_key(utxo_pos), Utxo.to_db_value(utxo)}} + + defp deposit_to_utxo(%{blknum: blknum, currency: cur, owner: owner, amount: amount}) do + {Utxo.position(blknum, 0, 0), %Utxo{amount: amount, currency: cur, owner: owner}} + end + + defp get_last_deposit_child_blknum([] = _deposits, current_height), do: current_height + + defp get_last_deposit_child_blknum(deposits, _current_height), + do: + deposits + |> Enum.max_by(& &1.blknum) + |> Map.get(:blknum) + + defp last_deposit_child_blknum_db_update([] = deposits, _last_deposit_child_blknum), do: deposits + + defp last_deposit_child_blknum_db_update(_deposits, last_deposit_child_blknum), + do: [{:put, :last_deposit_child_blknum, last_deposit_child_blknum}] + + defp find_utxo_matching_piggyback(%{tx_hash: tx_hash, output_index: oindex} = piggyback, utxos) do + # oindex in contract is 0-7 where 4-7 are outputs + oindex = oindex - 4 + + position = Enum.find(utxos, &match?({Utxo.position(_, _, ^oindex), %Utxo{creating_txhash: ^tx_hash}}, &1)) + + {piggyback, position} end end diff --git a/apps/omg/lib/omg/state/transaction/validator.ex b/apps/omg/lib/omg/state/transaction/validator.ex new file mode 100644 index 0000000000..98b4ab1dea --- /dev/null +++ b/apps/omg/lib/omg/state/transaction/validator.ex @@ -0,0 +1,91 @@ +# Copyright 2019 OmiseGO Pte Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +defmodule OMG.State.Transaction.Validator do + @moduledoc """ + Provides functions for transaction validation of transaction processing in OMG.State.Core. + + """ + + @maximum_block_size 65_536 + alias OMG.Fees + alias OMG.State.Core + alias OMG.State.Transaction + alias OMG.Utxo + require Utxo + + def validate_block_size(%Core{tx_index: number_of_transactions_in_block}) do + case number_of_transactions_in_block == @maximum_block_size do + true -> {:error, :too_many_transactions_in_block} + false -> :ok + end + end + + def correct_inputs?(%Core{utxos: utxos} = state, tx) do + inputs = Transaction.get_inputs(tx) + + with :ok <- inputs_not_from_future_block?(state, inputs), + {:ok, input_utxos} <- get_input_utxos(utxos, inputs), + input_utxos_owners <- Enum.map(input_utxos, fn %{owner: owner} -> owner end), + :ok <- Transaction.Recovered.all_spenders_authorized(tx, input_utxos_owners) do + {:ok, get_amounts_by_currency(input_utxos)} + end + end + + def get_amounts_by_currency(utxos) do + utxos + |> Enum.group_by(fn %{currency: currency} -> currency end, fn %{amount: amount} -> amount end) + |> Enum.map(fn {currency, amounts} -> {currency, Enum.sum(amounts)} end) + |> Map.new() + end + + def amounts_add_up?(input_amounts, output_amounts) do + for {output_currency, output_amount} <- Map.to_list(output_amounts) do + input_amount = Map.get(input_amounts, output_currency, 0) + input_amount >= output_amount + end + |> Enum.all?() + |> if(do: :ok, else: {:error, :amounts_do_not_add_up}) + end + + def transaction_covers_fee?(input_amounts, output_amounts, fees) do + Fees.covered?(input_amounts, output_amounts, fees) + |> if(do: :ok, else: {:error, :fees_not_covered}) + end + + defp inputs_not_from_future_block?(%Core{height: blknum}, inputs) do + no_utxo_from_future_block = + inputs + |> Enum.all?(fn Utxo.position(input_blknum, _, _) -> blknum >= input_blknum end) + + if no_utxo_from_future_block, do: :ok, else: {:error, :input_utxo_ahead_of_state} + end + + defp get_input_utxos(utxos, inputs) do + inputs + |> Enum.reduce_while({:ok, []}, fn input, acc -> get_utxos(utxos, input, acc) end) + |> reverse() + end + + defp get_utxos(utxos, position, {:ok, acc}) do + case Map.get(utxos, position) do + nil -> {:halt, {:error, :utxo_not_found}} + found -> {:cont, {:ok, [found | acc]}} + end + end + + @spec reverse({:ok, any()} | {:error, :utxo_not_found}) :: {:ok, list(any())} | {:error, :utxo_not_found} + defp reverse({:ok, input_utxos}), do: {:ok, Enum.reverse(input_utxos)} + defp reverse({:error, :utxo_not_found} = result), do: result +end From 5cbc4fdcad0e9127d983efd969dd3b32cc177efa Mon Sep 17 00:00:00 2001 From: Ino Murko Date: Fri, 7 Jun 2019 14:42:32 +0200 Subject: [PATCH 07/16] refactor: splitting the block apply execution into smaller bits --- .../lib/omg_watcher/block_getter.ex | 94 +++++++++++-------- 1 file changed, 55 insertions(+), 39 deletions(-) diff --git a/apps/omg_watcher/lib/omg_watcher/block_getter.ex b/apps/omg_watcher/lib/omg_watcher/block_getter.ex index 0968d38473..86f4d34ac3 100644 --- a/apps/omg_watcher/lib/omg_watcher/block_getter.ex +++ b/apps/omg_watcher/lib/omg_watcher/block_getter.ex @@ -107,55 +107,71 @@ defmodule OMG.Watcher.BlockGetter do {:noreply, state} end - @decorate measure_start() - def handle_cast( - {:apply_block, - %BlockApplication{ - transactions: transactions, - number: blknum, - eth_height: eth_height - } = to_apply}, - state - ) do - with {:ok, _} <- Core.chain_ok(state), - tx_exec_results = for(tx <- transactions, do: OMG.State.exec(tx, :ignore)), - {:ok, state} <- Core.validate_executions(tx_exec_results, to_apply, state) do - _ = - to_apply + def handle_continue({:execute_transactions, block_application}, state) do + tx_exec_results = for(tx <- block_application.transactions, do: OMG.State.exec(tx, :ignore)) + + case Core.validate_executions(tx_exec_results, block_application, state) do + {:ok, state} -> + block_application |> Core.ensure_block_imported_once(state) |> Enum.each(&DB.Transaction.update_with/1) - exit_processor_results = ExitProcessor.check_validity() + {:noreply, state, {:continue, {:run_block_download_task, block_application}}} - {state, synced_height, db_updates} = - state - |> run_block_download_task() - |> Core.consider_exits(exit_processor_results) - |> Core.apply_block(to_apply) + {{:error, _} = error, new_state} -> + :ok = update_status(new_state) + _ = Logger.error("Invalid block #{inspect(block_application.number)}, because of #{inspect(error)}") + {:noreply, new_state} + end + end - _ = Logger.debug("Synced height update: #{inspect(db_updates)}") + def handle_continue({:run_block_download_task, block_application}, state), + do: {:noreply, run_block_download_task(state), {:continue, {:close_and_apply_block, block_application}}} - {:ok, db_updates_from_state} = OMG.State.close_block(eth_height) - :ok = OMG.DB.multi_update(db_updates ++ db_updates_from_state) - :ok = check_in_to_coordinator(synced_height) - :ok = update_status(state) + def handle_continue({:close_and_apply_block, block_application}, state) do + {:ok, db_updates_from_state} = OMG.State.close_block(block_application.eth_height) - _ = - Logger.info( - "Applied block: \##{inspect(blknum)}, from eth height: #{inspect(eth_height)} " <> - "with #{inspect(length(transactions))} txs" - ) + {state, synced_height, db_updates} = Core.apply_block(state, block_application) - {:noreply, state} - else - {{:error, _} = error, new_state} -> - :ok = update_status(new_state) - _ = Logger.error("Invalid block #{inspect(blknum)}, because of #{inspect(error)}") - {:noreply, new_state} + _ = Logger.debug("Synced height update: #{inspect(db_updates)}") - {:error, _} = error -> + :ok = OMG.DB.multi_update(db_updates ++ db_updates_from_state) + :ok = check_in_to_coordinator(synced_height) + + _ = + Logger.info( + "Applied block: \##{inspect(block_application.number)}, from eth height: #{ + inspect(block_application.eth_height) + } " <> + "with #{inspect(length(block_application.transactions))} txs" + ) + + {:noreply, state, {:continue, :check_validity}} + end + + def handle_continue(:check_validity, state) do + exit_processor_results = ExitProcessor.check_validity() + state = Core.consider_exits(state, exit_processor_results) + :ok = update_status(state) + {:noreply, state} + end + + @decorate measure_start() + def handle_cast({:apply_block, %BlockApplication{} = block_application}, state) do + case Core.chain_ok(state) do + {:ok, _} -> + {:noreply, state, {:continue, {:execute_transactions, block_application}}} + + error -> :ok = update_status(state) - _ = Logger.warn("Chain already invalid before applying block #{inspect(blknum)} because of #{inspect(error)}") + + _ = + Logger.warn( + "Chain already invalid before applying block #{inspect(block_application.number)} because of #{ + inspect(error) + }" + ) + {:noreply, state} end end From 44d91ddffaab01214195359de5427fe00f85ff02 Mon Sep 17 00:00:00 2001 From: Ino Murko Date: Fri, 7 Jun 2019 19:14:55 +0200 Subject: [PATCH 08/16] fix: further state refactor --- apps/omg/lib/omg/state.ex | 3 +- apps/omg/lib/omg/state/core.ex | 35 ++++++------------ .../lib/omg/state/transaction/validator.ex | 36 +++++++++++++++---- .../lib/omg_watcher/api/account.ex | 1 - .../lib/omg_watcher/block_getter.ex | 18 +++++----- dialyzer.ignore-warnings | 7 ++-- 6 files changed, 57 insertions(+), 43 deletions(-) diff --git a/apps/omg/lib/omg/state.ex b/apps/omg/lib/omg/state.ex index 5b2878bf36..db5462f7fc 100644 --- a/apps/omg/lib/omg/state.ex +++ b/apps/omg/lib/omg/state.ex @@ -24,13 +24,14 @@ defmodule OMG.State do alias OMG.Recorder alias OMG.State.Core alias OMG.State.Transaction + alias OMG.State.Transaction.Validator alias OMG.Utxo use GenServer use OMG.Utils.Metrics use OMG.Utils.LoggerExt - @type exec_error :: Core.exec_error() + @type exec_error :: Validator.exec_error() ### Client diff --git a/apps/omg/lib/omg/state/core.ex b/apps/omg/lib/omg/state/core.ex index ed82db9b90..c1c4ff1eb2 100644 --- a/apps/omg/lib/omg/state/core.ex +++ b/apps/omg/lib/omg/state/core.ex @@ -65,12 +65,6 @@ defmodule OMG.State.Core do @type utxos() :: %{Utxo.Position.t() => Utxo.t()} - @type exec_error :: - :unauthorized_spent - | :amounts_do_not_add_up - | :invalid_current_block_number - | :utxo_not_found - @type deposit_event :: %{deposit: %{amount: non_neg_integer, owner: Crypto.address_t()}} @type tx_event :: %{ tx: Transaction.Recovered.t(), @@ -157,26 +151,19 @@ defmodule OMG.State.Core do """ @spec exec(state :: t(), tx :: Transaction.Recovered.t(), fees :: Fees.fee_t()) :: {:ok, {Transaction.tx_hash(), pos_integer, non_neg_integer}, t()} - | {{:error, exec_error}, t()} - def exec( - %Core{height: height, tx_index: tx_index} = state, - %Transaction.Recovered{} = tx, - fees - ) do + | {{:error, Validator.exec_error()}, t()} + def exec(%Core{} = state, %Transaction.Recovered{} = tx, fees) do tx_hash = Transaction.raw_txhash(tx) - outputs = Transaction.get_outputs(tx) - with :ok <- Validator.validate_block_size(state), - {:ok, input_amounts_by_currency} <- Validator.correct_inputs?(state, tx), - output_amounts_by_currency = Validator.get_amounts_by_currency(outputs), - :ok <- Validator.amounts_add_up?(input_amounts_by_currency, output_amounts_by_currency), - :ok <- Validator.transaction_covers_fee?(input_amounts_by_currency, output_amounts_by_currency, fees) do - {:ok, {tx_hash, height, tx_index}, - state - |> apply_spend(tx) - |> add_pending_tx(tx)} - else - {:error, _reason} = error -> {error, state} + case Validator.can_apply_spend(state, tx, fees) do + true -> + {:ok, {tx_hash, state.height, state.tx_index}, + state + |> apply_spend(tx) + |> add_pending_tx(tx)} + + {{:error, _reason}, _state} = error -> + error end end diff --git a/apps/omg/lib/omg/state/transaction/validator.ex b/apps/omg/lib/omg/state/transaction/validator.ex index 98b4ab1dea..9c9efc599b 100644 --- a/apps/omg/lib/omg/state/transaction/validator.ex +++ b/apps/omg/lib/omg/state/transaction/validator.ex @@ -14,7 +14,7 @@ defmodule OMG.State.Transaction.Validator do @moduledoc """ - Provides functions for transaction validation of transaction processing in OMG.State.Core. + Provides functions for stateful transaction validation for transaction processing in OMG.State.Core. """ @@ -25,14 +25,38 @@ defmodule OMG.State.Transaction.Validator do alias OMG.Utxo require Utxo - def validate_block_size(%Core{tx_index: number_of_transactions_in_block}) do + @type exec_error :: + :amounts_do_not_add_up + | :fees_not_covered + | :input_utxo_ahead_of_state + | :too_many_transactions_in_block + | :unauthorized_spent + | :utxo_not_found + + @spec can_apply_spend(state :: Core.t(), tx :: Transaction.Recovered.t(), fees :: Fees.fee_t()) :: + true | {{:error, exec_error()}, Core.t()} + def can_apply_spend(state, %Transaction.Recovered{} = tx, fees) do + outputs = Transaction.get_outputs(tx) + + with :ok <- validate_block_size(state), + {:ok, input_amounts_by_currency} <- correct_inputs?(state, tx), + output_amounts_by_currency = get_amounts_by_currency(outputs), + :ok <- amounts_add_up?(input_amounts_by_currency, output_amounts_by_currency), + :ok <- transaction_covers_fee?(input_amounts_by_currency, output_amounts_by_currency, fees) do + true + else + {:error, _reason} = error -> {error, state} + end + end + + defp validate_block_size(%Core{tx_index: number_of_transactions_in_block}) do case number_of_transactions_in_block == @maximum_block_size do true -> {:error, :too_many_transactions_in_block} false -> :ok end end - def correct_inputs?(%Core{utxos: utxos} = state, tx) do + defp correct_inputs?(%Core{utxos: utxos} = state, tx) do inputs = Transaction.get_inputs(tx) with :ok <- inputs_not_from_future_block?(state, inputs), @@ -43,14 +67,14 @@ defmodule OMG.State.Transaction.Validator do end end - def get_amounts_by_currency(utxos) do + defp get_amounts_by_currency(utxos) do utxos |> Enum.group_by(fn %{currency: currency} -> currency end, fn %{amount: amount} -> amount end) |> Enum.map(fn {currency, amounts} -> {currency, Enum.sum(amounts)} end) |> Map.new() end - def amounts_add_up?(input_amounts, output_amounts) do + defp amounts_add_up?(input_amounts, output_amounts) do for {output_currency, output_amount} <- Map.to_list(output_amounts) do input_amount = Map.get(input_amounts, output_currency, 0) input_amount >= output_amount @@ -59,7 +83,7 @@ defmodule OMG.State.Transaction.Validator do |> if(do: :ok, else: {:error, :amounts_do_not_add_up}) end - def transaction_covers_fee?(input_amounts, output_amounts, fees) do + defp transaction_covers_fee?(input_amounts, output_amounts, fees) do Fees.covered?(input_amounts, output_amounts, fees) |> if(do: :ok, else: {:error, :fees_not_covered}) end diff --git a/apps/omg_watcher/lib/omg_watcher/api/account.ex b/apps/omg_watcher/lib/omg_watcher/api/account.ex index 2d6e9ff140..2656722e81 100644 --- a/apps/omg_watcher/lib/omg_watcher/api/account.ex +++ b/apps/omg_watcher/lib/omg_watcher/api/account.ex @@ -38,7 +38,6 @@ defmodule OMG.Watcher.API.Account do @doc """ Gets all utxos belonging to the given address. Slow operation, compatible with security-critical. - TODO: what's the actuall problem here? Storage model? """ @spec get_exitable_utxos(OMG.Crypto.address_t()) :: list(OMG.State.Core.exitable_utxos()) def get_exitable_utxos(address) do diff --git a/apps/omg_watcher/lib/omg_watcher/block_getter.ex b/apps/omg_watcher/lib/omg_watcher/block_getter.ex index 86f4d34ac3..4033946c42 100644 --- a/apps/omg_watcher/lib/omg_watcher/block_getter.ex +++ b/apps/omg_watcher/lib/omg_watcher/block_getter.ex @@ -107,7 +107,7 @@ defmodule OMG.Watcher.BlockGetter do {:noreply, state} end - def handle_continue({:execute_transactions, block_application}, state) do + def handle_continue({:apply_block_step, :execute_transactions, block_application}, state) do tx_exec_results = for(tx <- block_application.transactions, do: OMG.State.exec(tx, :ignore)) case Core.validate_executions(tx_exec_results, block_application, state) do @@ -116,7 +116,7 @@ defmodule OMG.Watcher.BlockGetter do |> Core.ensure_block_imported_once(state) |> Enum.each(&DB.Transaction.update_with/1) - {:noreply, state, {:continue, {:run_block_download_task, block_application}}} + {:noreply, state, {:continue, {:apply_block_step, :run_block_download_task, block_application}}} {{:error, _} = error, new_state} -> :ok = update_status(new_state) @@ -125,10 +125,12 @@ defmodule OMG.Watcher.BlockGetter do end end - def handle_continue({:run_block_download_task, block_application}, state), - do: {:noreply, run_block_download_task(state), {:continue, {:close_and_apply_block, block_application}}} + def handle_continue({:apply_block_step, :run_block_download_task, block_application}, state), + do: + {:noreply, run_block_download_task(state), + {:continue, {:apply_block_step, :close_and_apply_block, block_application}}} - def handle_continue({:close_and_apply_block, block_application}, state) do + def handle_continue({:apply_block_step, :close_and_apply_block, block_application}, state) do {:ok, db_updates_from_state} = OMG.State.close_block(block_application.eth_height) {state, synced_height, db_updates} = Core.apply_block(state, block_application) @@ -146,10 +148,10 @@ defmodule OMG.Watcher.BlockGetter do "with #{inspect(length(block_application.transactions))} txs" ) - {:noreply, state, {:continue, :check_validity}} + {:noreply, state, {:continue, {:apply_block_step, :check_validity}}} end - def handle_continue(:check_validity, state) do + def handle_continue({:apply_block_step, :check_validity}, state) do exit_processor_results = ExitProcessor.check_validity() state = Core.consider_exits(state, exit_processor_results) :ok = update_status(state) @@ -160,7 +162,7 @@ defmodule OMG.Watcher.BlockGetter do def handle_cast({:apply_block, %BlockApplication{} = block_application}, state) do case Core.chain_ok(state) do {:ok, _} -> - {:noreply, state, {:continue, {:execute_transactions, block_application}}} + {:noreply, state, {:continue, {:apply_block_step, :execute_transactions, block_application}}} error -> :ok = update_status(state) diff --git a/dialyzer.ignore-warnings b/dialyzer.ignore-warnings index 5b2fff9197..9b8085895c 100644 --- a/dialyzer.ignore-warnings +++ b/dialyzer.ignore-warnings @@ -33,7 +33,7 @@ Type specification 'Elixir.OMG.ChildChain.BlockQueue.Core':adjust_gas_price('Eli Type specification 'Elixir.OMG.ChildChain.BlockQueue.Core':calculate_gas_price('Elixir.OMG.ChildChain.BlockQueue.Core':t()) -> pos_integer() is not equal to the success typing: 'Elixir.OMG.ChildChain.BlockQueue.Core':calculate_gas_price(#{'__struct__':='Elixir.OMG.ChildChain.BlockQueue.Core', 'blocks':=#{pos_integer()=>map()}, 'chain_start_parent_height':=pos_integer(), 'child_block_interval':=pos_integer(), 'finality_threshold':=pos_integer(), 'formed_child_block_num':=non_neg_integer(), 'gas_price_adj_params':=#{'__struct__':='Elixir.OMG.ChildChain.BlockQueue.GasPriceAdjustment', 'eth_gap_without_child_blocks':=pos_integer(), 'gas_price_lowering_factor':=float(), 'gas_price_raising_factor':=float(), 'last_block_mined':={_,_}, 'max_gas_price':=pos_integer()}, 'gas_price_to_use':=pos_integer(), 'last_enqueued_block_at_height':=pos_integer(), 'last_parent_height':=_, 'mined_child_block_num':=non_neg_integer(), 'minimal_enqueue_block_gap':=pos Type specification 'Elixir.OMG.ChildChain.BlockQueue.Core':child_block_nums_to_init_with(non_neg_integer(),non_neg_integer(),pos_integer(),non_neg_integer()) -> [any()] is not equal to the success typing: 'Elixir.OMG.ChildChain.BlockQueue.Core':child_block_nums_to_init_with(number(),_,number(),number()) -> [integer()] Type specification 'Elixir.OMG.ChildChain.BlockQueue.Core':enqueue_block('Elixir.OMG.ChildChain.BlockQueue.Core':t(),'Elixir.OMG.ChildChain.BlockQueue':hash(),'Elixir.OMG.ChildChain.BlockQueue':plasma_block_num(),pos_integer()) -> 'Elixir.OMG.ChildChain.BlockQueue.Core':t() | {'error','unexpected_block_number'} is not equal to the success typing: 'Elixir.OMG.ChildChain.BlockQueue.Core':enqueue_block(atom() | #{'child_block_interval':=number(), 'formed_child_block_num':=number(), _=>_},_,_,_) -> {'error','unexpected_block_number'} | #{'blocks':=map(), 'child_block_interval':=number(), 'formed_child_block_num':=number(), 'last_enqueued_block_at_height':=_, 'wait_for_enqueue':='false', _=>_} -Type specification 'Elixir.OMG.ChildChain.BlockQueue.Core':enqueue_existing_blocks('Elixir.OMG.ChildChain.BlockQueue.Core':t(),'Elixir.OMG.ChildChain.BlockQueue':hash(),[{pos_integer(),'Elixir.OMG.ChildChain.BlockQueue':hash()}]) -> {'ok','Elixir.OMG.ChildChain.BlockQueue.Core':t()} | {'error','contract_ahead_of_db' | 'mined_blknum_not_found_in_db' | 'hashes_dont_match'} is not equal to the success typing: 'Elixir.OMG.ChildChain.BlockQueue.Core':enqueue_existing_blocks(#{'__struct__':='Elixir.OMG.ChildChain.BlockQueue.Core', 'blocks':=#{}, 'chain_start_parent_height':=_, 'child_block_interval':=_, 'finality_threshold':=_, 'formed_child_block_num':=0, 'gas_price_adj_params':=#{'__struct__':='Elixir.OMG.ChildChain.BlockQueue.GasPriceAdjustment', 'eth_gap_without_child_blocks':=2, 'gas_price_lowering_factor':=float(), 'gas_price_raising_factor':=float(), 'last_block_mined':='nil', 'max_gas_price':=20000000000}, 'gas_price_to_use':=20000000000, +Type specification 'Elixir.OMG.ChildChain.BlockQueue.Core':enqueue_existing_blocks('Elixir.OMG.ChildChain.BlockQueue.Core':t(),'Elixir.OMG.ChildChain.BlockQueue':hash(),[{pos_integer(),'Elixir.OMG.ChildChain.BlockQueue':hash()}]) -> {'ok','Elixir.OMG.ChildChain.BlockQueue.Core':t()} | {'error','contract_ahead_of_db' | 'mined_blknum_not_found_in_db' | 'hashes_dont_match'} is not equal to the success typing: 'Elixir.OMG.ChildChain.BlockQueue.Core':enqueue_existing_blocks(#{'__struct__':='Elixir.OMG.ChildChain.BlockQueue.Core', 'blocks':=#{}, 'chain_start_parent_height':=_, 'child_block_interval':=_, 'finality_threshold':=_, 'formed_child_block_num':=0, 'gas_price_adj_params':=#{'__struct__':='Elixir.OMG.ChildChain.BlockQueue.GasPriceAdjustment', 'eth_gap_without_child_blocks':=2, 'gas_price_lowering_factor':=float(), 'gas_price_raising_factor':=float(), 'last_block_mined':='nil', 'max_gas_price':=20000000000}, 'gas_price_to_use':=20000000000, Type specification 'Elixir.OMG.ChildChain.BlockQueue.Core':get_blocks_to_submit('Elixir.OMG.ChildChain.BlockQueue.Core':t()) -> ['Elixir.OMG.ChildChain.BlockQueue':encoded_signed_tx()] is a subtype of the success typing: 'Elixir.OMG.ChildChain.BlockQueue.Core':get_blocks_to_submit(#{'__struct__':='Elixir.OMG.ChildChain.BlockQueue.Core', 'blocks':=#{pos_integer()=>map()}, 'chain_start_parent_height':=pos_integer(), 'child_block_interval':=pos_integer(), 'finality_threshold':=pos_integer(), 'formed_child_block_num':=non_neg_integer(), 'gas_price_adj_params':=#{'__struct__':='Elixir.OMG.ChildChain.BlockQueue.GasPriceAdjustment', 'eth_gap_without_child_blocks':=pos_integer(), 'gas_price_lowering_factor':=float(), 'gas_price_raising_factor':=float(), 'last_block_mined':='nil' | tuple(), 'max_gas_price':=pos_integer()}, 'gas_price_to_use':=pos_integer(), 'last_enqueued_block_at_height':=pos_integer(), 'last_parent_height':=_, 'mined_child_block_nu Type specification 'Elixir.OMG.ChildChain.BlockQueue.Core':new(elixir:keyword()) -> {'ok','Elixir.OMG.ChildChain.BlockQueue.Core':t()} | {'error','mined_hash_not_found_in_db'} | {'error','contract_ahead_of_db'} is not equal to the success typing: 'Elixir.OMG.ChildChain.BlockQueue.Core':new([{'chain_start_parent_height' | 'child_block_interval' | 'finality_threshold' | 'known_hashes' | 'last_enqueued_block_at_height' | 'mined_child_block_num' | 'minimal_enqueue_block_gap' | 'parent_height' | 'top_mined_hash','nil' | <<_:256>> | [{_,_}] | non_neg_integer()},...]) -> {'error','contract_ahead_of_db' | 'hashes_dont_match' | 'mined_blknum_not_found_in_db'} | {'ok',#{'__struct__':='Elixir.OMG.ChildChain.BlockQueue.Core', 'blocks':=#{pos_integer()=>map()}, 'chain_start_parent_height':=pos_integer(), 'child_block_interval':=pos_integer(), 'finality_threshold':=pos_integer(), 'formed_child_block_num':=non_neg_integer(), 'gas_price_adj_params':=#{'__st Type specification 'Elixir.OMG.ChildChain.BlockQueue.Core':process_submit_result('Elixir.OMG.ChildChain.BlockQueue.Core.BlockSubmission':t(),submit_result_t(),'Elixir.OMG.ChildChain.BlockQueue.Core.BlockSubmission':plasma_block_num()) -> 'ok' | {'error',atom()} is not equal to the success typing: 'Elixir.OMG.ChildChain.BlockQueue.Core':process_submit_result(_,{'error',map()} | {'ok',_},_) -> 'ok' | {'error','account_locked' | 'nonce_too_low'} @@ -98,13 +98,14 @@ Type specification 'Elixir.OMG.Signature':recover_public(keccak_hash(),hash_v(), Type specification 'Elixir.OMG.State':'utxo_exists?'('Elixir.OMG.Utxo.Position':t()) -> boolean() is a subtype of the success typing: 'Elixir.OMG.State':'utxo_exists?'(_) -> any() Type specification 'Elixir.OMG.State':close_block(pos_integer()) -> {'ok',['Elixir.OMG.State.Core':db_update()]} is a subtype of the success typing: 'Elixir.OMG.State':close_block(_) -> any() Type specification 'Elixir.OMG.State':deposit(deposits::['Elixir.OMG.State.Core':deposit()]) -> {'ok',['Elixir.OMG.State.Core':db_update()]} is a subtype of the success typing: 'Elixir.OMG.State':deposit(_) -> any() +Type specification 'Elixir.OMG.State.Transaction.Validator':can_apply_spend(state::'Elixir.OMG.State.Core':t(),tx::'Elixir.OMG.State.Transaction.Recovered':t(),fees::'Elixir.OMG.Fees':fee_t()) -> 'true' | {{'error',exec_error()},'Elixir.OMG.State.Core':t()} is a subtype of the success typing: 'Elixir.OMG.State.Transaction.Validator':can_apply_spend(#{'__struct__':='Elixir.OMG.State.Core', 'tx_index':=_, _=>_},#{'__struct__':='Elixir.OMG.State.Transaction.Recovered', 'inputs'=>[#{'blknum':=non_neg_integer(), 'oindex':=non_neg_integer(), 'txindex':=non_neg_integer()}], 'metadata'=>'nil' | binary(), 'outputs'=>[#{'amount':=non_neg_integer(), 'currency':=<<_:160>>, 'owner':=<<_:160>>}], 'raw_tx'=>#{'__struct__':='Elixir.OMG.State.Transaction', 'inputs':=[map()], 'metadata':='nil' | binary(), 'outputs':=[map()]}, 'signed_tx'=>#{'__struct__':='Elixir.OMG.State.Transaction.Signed', 'raw_tx':=#{'__struct__':='Elixir.OMG.State.Transaction', 'inputs':=[any()], 'metadata':='nil' | binary(), 'outputs':=[any()]}, 'signed_tx_bytes':='nil' | binary(), 'sigs':=[<<_:520>>]}, 'signed_tx_bytes'=>'nil' | binary(), 'sigs'=>[<<_:520>>], 'spenders'=>[<<_:160>>], 'tx_hash'=><<_:256>>},_) -> 'true' | {{'error','amounts_do_not_add_up' | 'fees_not_covered' | 'input_utxo_ahead_of_state' | 'too_many_transactions_in_block' | 'unauthorized_spent' | 'utxo_not_found'},#{'__struct__':='Elixir.OMG.State.Core', 'tx_index':=_, _=>_}} Type specification 'Elixir.OMG.State':exec(tx::'Elixir.OMG.State.Transaction.Recovered':t(),fees::'Elixir.OMG.Fees':fee_t()) -> {'ok',{'Elixir.OMG.State.Transaction':tx_hash(),pos_integer(),non_neg_integer()}} | {'error',exec_error()} is a subtype of the success typing: 'Elixir.OMG.State':exec(_,_) -> any() Type specification 'Elixir.OMG.State':exit_utxos(utxos::'Elixir.OMG.State.Core':exiting_utxos_t()) -> {'ok',['Elixir.OMG.State.Core':db_update()],'Elixir.OMG.State.Core':validities_t()} is a subtype of the success typing: 'Elixir.OMG.State':exit_utxos(_) -> any() Type specification 'Elixir.OMG.State':get_status() -> {non_neg_integer(),boolean()} is a subtype of the success typing: 'Elixir.OMG.State':get_status() -> any() Type specification 'Elixir.OMG.State.Core':'utxo_exists?'('Elixir.OMG.Utxo.Position':t(),t()) -> boolean() is a subtype of the success typing: 'Elixir.OMG.State.Core':'utxo_exists?'({'utxo_position',_,_,_},#{'__struct__':='Elixir.OMG.State.Core', 'utxos':=map(), _=>_}) -> boolean() Type specification 'Elixir.OMG.State.Core':db_update_utxos(non_neg_integer(),['Elixir.OMG.State.Transaction.Recovered':t()]) -> [{'put','utxo',{'Elixir.OMG.Utxo.Position':db_t(),'Elixir.OMG.Utxo':t()}} | {'delet','utxo','Elixir.OMG.Utxo.Position':db_t()}] is a subtype of the success typing: 'Elixir.OMG.State.Core':db_update_utxos(non_neg_integer(),[#{'__struct__':='Elixir.OMG.State.Transaction.Recovered', 'signed_tx':=map(), 'spenders':=[any()], 'tx_hash':=<<_:256>>}]) -> maybe_improper_list() Type specification 'Elixir.OMG.State.Core':deposit(deposits::[deposit()],state::t()) -> {'ok',{[deposit_event()],[db_update()]},new_state::t()} is a subtype of the success typing: 'Elixir.OMG.State.Core':deposit(_,#{'__struct__':='Elixir.OMG.State.Core', 'last_deposit_child_blknum':=_, 'utxos':=map(), _=>_}) -> {'ok',{[any()],[any()]},#{'__struct__':='Elixir.OMG.State.Core', 'last_deposit_child_blknum':=_, 'utxos':=map(), _=>_}} -Type specification 'Elixir.OMG.State.Core':exec(state::t(),tx::'Elixir.OMG.State.Transaction.Recovered':t(),fees::'Elixir.OMG.Fees':fee_t()) -> {'ok',{'Elixir.OMG.State.Transaction':tx_hash(),pos_integer(),non_neg_integer()},t()} | {{'error',exec_error()},t()} is not equal to the success typing: 'Elixir.OMG.State.Core':exec(#{'__struct__':='Elixir.OMG.State.Core', 'height':=_, 'tx_index':=_, _=>_},#{'__struct__':='Elixir.OMG.State.Transaction.Recovered', 'inputs'=>[#{'blknum':=non_neg_integer(), 'oindex':=non_neg_integer(), 'txindex':=non_neg_integer()}], 'metadata'=>'nil' | binary(), 'outputs'=>[#{'amount':=non_neg_integer(), 'currency':=<<_:160>>, 'owner':=<<_:160>>}], 'raw_tx'=>#{'__struct__':='Elixir.OMG.State.Transaction', 'inputs':=[map()], 'metadata':='nil' | binary(), 'outputs':=[map()]}, 'signed_tx'=>#{'__struct__':='Elixir.OMG.State.Transaction.Signed', 'raw_tx':=#{'__struct__':='Elixir.OMG.State.Transaction', 'inputs':=[any()], 'metadata':='nil' | binary(), 'o +Type specification 'Elixir.OMG.State.Core':exec(state::t(),tx::'Elixir.OMG.State.Transaction.Recovered':t(),fees::'Elixir.OMG.Fees':fee_t()) -> {'ok',{'Elixir.OMG.State.Transaction':tx_hash(),pos_integer(),non_neg_integer()},t()} | {{'error','Elixir.OMG.State.Transaction.Validator':exec_error()},t()} is not equal to the success typing: 'Elixir.OMG.State.Core':exec(#{'__struct__':='Elixir.OMG.State.Core', 'height':=non_neg_integer(), 'last_deposit_child_blknum':=non_neg_integer(), 'pending_txs':=[#{'__struct__':='Elixir.OMG.State.Transaction.Recovered', 'signed_tx':=map(), 'spenders':=[any()], 'tx_hash':=<<_:256>>}], 'tx_index':=non_neg_integer(), 'utxos':=#{{'utxo_position',non_neg_integer(),non_neg_integer(),non_neg_integer()}=>#{'__struct__':='Elixir.OMG.Utxo', 'amount':=non_neg_integer(), 'creating_txhash':=<<_:256>>, 'currency':=<<_:160>>, 'owner':=<<_:160>>}}},#{'__struct__':='Elixir.OMG.State.Transaction.Recovered', 'signed_tx':=#{'__struct__':='Elixir.OMG.State.Transaction.Signed', 'raw_tx':=#{'__struct__':='Elixir.OMG.State.Transaction', 'inputs':=[any()], 'metadata':='nil' | binary(), 'outputs':=[any()]}, 'signed_tx_bytes':='nil' | binary(), 'sigs':=[<<_:520>>]}, 'spenders':=[<<_:160>>], 'tx_hash':=<<_:256>>},'ignore' | #{<<_:160>>=>non_neg_integer()}) -> {{'error','amounts_do_not_add_up' | 'fees_not_covered' | 'input_utxo_ahead_of_state' | 'too_many_transactions_in_block' | 'unauthorized_spent' | 'utxo_not_found'},#{'__struct__':='Elixir.OMG.State.Core', 'height':=non_neg_integer(), 'last_deposit_child_blknum':=non_neg_integer(), 'pending_txs':=[map()], 'tx_index':=non_neg_integer(), 'utxos':=#{{_,_,_,_}=>map()}}} | {'ok',{<<_:256>>,non_neg_integer(),non_neg_integer()},#{'__struct__':='Elixir.OMG.State.Core', 'height':=non_neg_integer(), 'last_deposit_child_blknum':=non_neg_integer(), 'pending_txs':=[map(),...], 'tx_index':=pos_integer(), 'utxos':=map()}} Type specification 'Elixir.OMG.State.Core':exit_utxos(exiting_utxos::exiting_utxos_t(),state::t()) -> {'ok',{[db_update()],validities_t()},new_state::t()} is a subtype of the success typing: 'Elixir.OMG.State.Core':exit_utxos(_,#{'__struct__':='Elixir.OMG.State.Core', 'utxos':=map(), _=>_}) -> {'ok',{[any()],{_,_}},#{'__struct__':='Elixir.OMG.State.Core', 'utxos':=map(), _=>_}} Type specification 'Elixir.OMG.State.Core':extract_initial_state(utxos_query_result::[[{'Elixir.OMG.DB':utxo_pos_db_t(),'Elixir.OMG.Utxo':t()}]],height_query_result::non_neg_integer() | 'not_found',last_deposit_child_blknum_query_result::non_neg_integer() | 'not_found',child_block_interval::pos_integer()) -> {'ok',t()} | {'error','last_deposit_not_found' | 'top_block_number_not_found'} is not equal to the success typing: 'Elixir.OMG.State.Core':extract_initial_state(_,_,_,_) -> {'error','last_deposit_not_found' | 'top_block_number_not_found'} | {'ok',#{'__struct__':='Elixir.OMG.State.Core', 'height':=integer(), 'last_deposit_child_blknum':=integer(), 'pending_txs':=[], 'tx_index':=0, 'utxos':=map()}} Type specification 'Elixir.OMG.State.Core':form_block(pos_integer(),pos_integer() | 'nil',state::t()) -> {'ok',{'Elixir.OMG.Block':t(),[tx_event()],[db_update()]},new_state::t()} is not equal to the success typing: 'Elixir.OMG.State.Core':form_block(number(),_,#{'__struct__':='Elixir.OMG.State.Core', 'height':=non_neg_integer(), 'pending_txs':=_, 'tx_index':=_, _=>_}) -> {'ok',{map(),[any()],[any(),...]},#{'__struct__':='Elixir.OMG.State.Core', 'height':=number(), 'pending_txs':=[], 'tx_index':=0, _=>_}} @@ -194,7 +195,7 @@ Type specification 'Elixir.OMG.Watcher.ExitProcessor':get_competitor_for_ife(bin Type specification 'Elixir.OMG.Watcher.ExitProcessor':get_input_challenge_data('Elixir.OMG.State.Transaction.Signed':tx_bytes(),'Elixir.OMG.State.Transaction':input_index_t()) -> {'ok','Elixir.OMG.Watcher.ExitProcessor.Core':input_challenge_data()} | {'error','Elixir.OMG.Watcher.ExitProcessor.Core':piggyback_challenge_data_error()} is a subtype of the success typing: 'Elixir.OMG.Watcher.ExitProcessor':get_input_challenge_data(_,_) -> any() Type specification 'Elixir.OMG.Watcher.ExitProcessor':get_output_challenge_data('Elixir.OMG.State.Transaction.Signed':tx_bytes(),'Elixir.OMG.State.Transaction':input_index_t()) -> {'ok','Elixir.OMG.Watcher.ExitProcessor.Core':output_challenge_data()} | {'error','Elixir.OMG.Watcher.ExitProcessor.Core':piggyback_challenge_data_error()} is a subtype of the success typing: 'Elixir.OMG.Watcher.ExitProcessor':get_output_challenge_data(_,_) -> any() Type specification 'Elixir.OMG.Watcher.ExitProcessor':prove_canonical_for_ife(binary()) -> {'ok','Elixir.OMG.Watcher.ExitProcessor.Core':prove_canonical_data_t()} | {'error','no_viable_canonical_proof_found'} is a subtype of the success typing: 'Elixir.OMG.Watcher.ExitProcessor':prove_canonical_for_ife(_) -> any() -Type specification 'Elixir.OMG.Watcher.ExitProcessor':update_with_ife_txs_from_blocks('Elixir.OMG.Watcher.ExitProcessor.Core':t()) -> 'Elixir.OMG.Watcher.ExitProcessor.Core':t() is a subtype of the success typing: 'Elixir.OMG.Watcher.ExitProcessor':update_with_ife_txs_from_blocks(#{'__struct__':='Elixir.OMG.Watcher.ExitProcessor.Core', 'competitors':=#{<<_:256>>=>#{'__struct__':='Elixir.OMG.Watcher.ExitProcessor.CompetitorInfo', 'competing_input_index':=0 | 1 | 2 | 3, 'competing_input_signature':=<<_:520>>, 'tx':=map()}}, 'exits':=#{{'utxo_position',non_neg_integer(),non_neg_integer(),non_neg_integer()}=>#{'__struct__':='Elixir.OMG.Watcher.ExitProcessor.ExitInfo', 'amount':=non_neg_integer(), 'currency':=<<_:160>>, 'eth_height':=pos_integer(), 'is_active':=boolean(), 'owner':=<<_:160>>}}, 'in_flight_exits':=#{<<_:256>>=>#{'__struct__':='Elixir.OMG.Watcher.ExitProcessor.InFlightExitInfo', 'contract_id':=<<_:192>>, 'contract_tx_pos':='nil' | {_,_,_,_}, +Type specification 'Elixir.OMG.Watcher.ExitProcessor':update_with_ife_txs_from_blocks('Elixir.OMG.Watcher.ExitProcessor.Core':t()) -> 'Elixir.OMG.Watcher.ExitProcessor.Core':t() is a subtype of the success typing: 'Elixir.OMG.Watcher.ExitProcessor':update_with_ife_txs_from_blocks(#{'__struct__':='Elixir.OMG.Watcher.ExitProcessor.Core', 'competitors':=#{<<_:256>>=>#{'__struct__':='Elixir.OMG.Watcher.ExitProcessor.CompetitorInfo', 'competing_input_index':=0 | 1 | 2 | 3, 'competing_input_signature':=<<_:520>>, 'tx':=map()}}, 'exits':=#{{'utxo_position',non_neg_integer(),non_neg_integer(),non_neg_integer()}=>#{'__struct__':='Elixir.OMG.Watcher.ExitProcessor.ExitInfo', 'amount':=non_neg_integer(), 'currency':=<<_:160>>, 'eth_height':=pos_integer(), 'is_active':=boolean(), 'owner':=<<_:160>>}}, 'in_flight_exits':=#{<<_:256>>=>#{'__struct__':='Elixir.OMG.Watcher.ExitProcessor.InFlightExitInfo', 'contract_id':=<<_:192>>, 'contract_tx_pos':='nil' | {_,_,_,_}, Type specification 'Elixir.OMG.Watcher.ExitProcessor.Core':challenge_exits(t(),[map()]) -> {t(),[any()]} is a subtype of the success typing: 'Elixir.OMG.Watcher.ExitProcessor.Core':challenge_exits(#{'__struct__':='Elixir.OMG.Watcher.ExitProcessor.Core', 'exits':=map(), _=>_},_) -> {#{'__struct__':='Elixir.OMG.Watcher.ExitProcessor.Core', 'exits':=map(), _=>_},[any()]} Type specification 'Elixir.OMG.Watcher.ExitProcessor.Core':challenge_piggybacks(t(),[map()]) -> {t(),[any()]} is a subtype of the success typing: 'Elixir.OMG.Watcher.ExitProcessor.Core':challenge_piggybacks(#{'__struct__':='Elixir.OMG.Watcher.ExitProcessor.Core', _=>_},_) -> {#{'__struct__':='Elixir.OMG.Watcher.ExitProcessor.Core', 'in_flight_exits':=map(), _=>_},[any()]} Type specification 'Elixir.OMG.Watcher.ExitProcessor.Core':check_validity('Elixir.OMG.Watcher.ExitProcessor.Request':t(),t()) -> check_validity_result_t() is not equal to the success typing: 'Elixir.OMG.Watcher.ExitProcessor.Core':check_validity(#{'__struct__':='Elixir.OMG.Watcher.ExitProcessor.Request', 'blknum_now':='nil' | pos_integer(), 'blknums_to_get':=[pos_integer()], 'blocks_result':=[map()], 'eth_height_now':=pos_integer(), 'ife_input_spending_blocks_result':=[map()], 'ife_input_spends_to_get':=[{_,_,_,_}], 'ife_input_utxo_exists_result':=[boolean()], 'ife_input_utxos_to_check':=[{_,_,_,_}], 'piggybacked_blknums_to_get':=[pos_integer()], 'se_creating_blocks_result':=[map()], 'se_creating_blocks_to_get':=[pos_integer()], 'se_exit_id_result':='nil' | pos_integer(), 'se_exit_id_to_get':='nil' | binary(), 'se_exiting_pos':='nil' | {'utxo_position',non_neg_integer(),non_neg_integer(),non_neg_integer()}, 'se_spending_blocks_result':=[map()], ' From 9dbbac0a68d4961d9197cababebcb3e5c0b756e6 Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Mon, 10 Jun 2019 10:26:48 -0600 Subject: [PATCH 09/16] Update installation instructions for Linux --- docs/install.md | 45 +++++++++++++++++++++++++++------- docs/manual_service_startup.md | 18 ++++++-------- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/docs/install.md b/docs/install.md index 8e46e62c6b..8ac403ff3d 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,7 +1,4 @@ -# 1. Installation via Vagrant -Refer to https://github.com/omisego/xomg-vagrant. - -# 2. Full installation +# Full Installation **NOTE**: Currently the child chain server and watcher are bundled within a single umbrella app. @@ -10,15 +7,34 @@ Only **Linux** platforms are supported now. These instructions have been tested ## Prerequisites * **Erlang OTP** `>=20` (check with `elixir --version`) * **Elixir** `>=1.6` (check with `elixir --version`) -* **solc** `>=0.4.24` (check with `solc --version`) +* **solc** `~>0.4.24` (check with `solc --version`) ### Optional prerequisites * **`httpie`** - to run HTTP requests from `docs/demoxx.md` demos ## Install prerequisite packages + ``` sudo apt-get update -sudo apt-get -y install build-essential autoconf libtool libgmp3-dev libssl-dev wget git +sudo apt-get -y install \ + autoconf \ + build-essential \ + cmake \ + git \ + libgmp3-dev \ + libsecp256k1-dev \ + libssl-dev \ + libtool \ + wget +``` + +## Install PostgreSQL + +``` +sudo apt-get install postgresql postgresql-contrib +sudo -u postgres createuser omisego_dev +sudo -u postgres psql -c "alter user omisego_dev with encrypted password 'omisego_dev'" +sudo -u postgres psql -c "alter user omisego_dev CREATEDB" ``` ## Install Erlang @@ -36,7 +52,6 @@ sudo apt-get install -y esl-erlang sudo apt-get -y install elixir ``` - ## Install Geth ``` sudo apt-get install -y software-properties-common @@ -45,12 +60,23 @@ sudo apt-get update sudo apt-get -y install geth ``` -### Installing Parity -[Parity]((https://www.parity.io/ethereum/)) is supported. To use it, download the binary and put it into your PATH. +## Installing Parity +Parity is supported. To use it, download the [lastest stable +binary](https://www.parity.io/ethereum/#download) and put it into your PATH. + +``` +wget https://releases.parity.io/ethereum/v2.4.6/x86_64-unknown-linux-gnu/parity +chmod +x parity +sudo mv parity /usr/bin/ +``` ## Install solc ``` sudo apt-get install libssl-dev solc +wget https://github.com/ethereum/solidity/releases/download/v0.4.26/solidity-ubuntu-trusty.zip +unzip solidity-ubuntu-trusty.zip +sudo install solc /usr/local/bin +rm solc lllc solidity-ubuntu-trusty.zip ``` ## Install hex and rebar @@ -67,6 +93,7 @@ git clone https://github.com/omisego/elixir-omg ``` cd elixir-omg mix deps.get +mix deps.compile ``` ## Check this works! diff --git a/docs/manual_service_startup.md b/docs/manual_service_startup.md index bc50fcc695..ab95e59a26 100644 --- a/docs/manual_service_startup.md +++ b/docs/manual_service_startup.md @@ -1,5 +1,12 @@ # Manual steps to start the services -This process is intended for users who wish to start services manually, perhaps as part of a non-Docker deployment on a Linux host. + +This process is intended for users who wish to start services manually, perhaps +as part of a non-Docker deployment on a Linux host. + +## Installation + +First, install all dependecies using the the [installation +instructions](install.md). ## Setup The setup process for the Child chain server and for the Watcher is similar. @@ -139,15 +146,6 @@ mix xomg.child_chain.start --config ~/config.exs This assumes that you've got a developer environment Child chain server set up and running on the default `localhost:9656`, see above. -#### Configure the PostgreSQL server with: - -```bash -sudo -u postgres createuser omisego_dev -sudo -u postgres psql -alter user omisego_dev with encrypted password 'omisego_dev'; -ALTER USER omisego_dev CREATEDB; -``` - #### Configure the Watcher Copy the configuration file used by the Child chain server to `~/config_watcher.exs` From 4592b31f27dfe200d1aaec2705841d8d3426baee Mon Sep 17 00:00:00 2001 From: Chris Hunt Date: Mon, 10 Jun 2019 11:06:18 -0600 Subject: [PATCH 10/16] Relax solc version to ~>0.4 cc @achiurizo --- docs/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install.md b/docs/install.md index 8ac403ff3d..4b2b26269e 100644 --- a/docs/install.md +++ b/docs/install.md @@ -7,7 +7,7 @@ Only **Linux** platforms are supported now. These instructions have been tested ## Prerequisites * **Erlang OTP** `>=20` (check with `elixir --version`) * **Elixir** `>=1.6` (check with `elixir --version`) -* **solc** `~>0.4.24` (check with `solc --version`) +* **solc** `~>0.4` (check with `solc --version`) ### Optional prerequisites * **`httpie`** - to run HTTP requests from `docs/demoxx.md` demos From 2b175783e34263024d62560f33eaaf12ddfe4e86 Mon Sep 17 00:00:00 2001 From: jbunce Date: Wed, 12 Jun 2019 14:34:28 +0700 Subject: [PATCH 11/16] feat: Add functional tests to master commits --- .circleci/config.yml | 5 ++ .circleci/test_runner.py | 104 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100755 .circleci/test_runner.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 04fbc9ec6c..961a76f576 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -318,6 +318,11 @@ jobs: kubectl set image statefulset childchain-samrong childchain=$DOCKER_IMAGE while true; do if [ "$(kubectl get pods childchain-samrong-0 -o jsonpath=\"{.status.phase}\" | grep Running)" ]; then break; fi; done kubectl set image statefulset watcher-samrong watcher=$DOCKER_IMAGE + sleep(120) # TODO: Get the readiness status from each statefulset + - run: + name: Functional Tests + command: | + python3 .circleci/test_runner.py build_and_deploy_staging: docker: diff --git a/.circleci/test_runner.py b/.circleci/test_runner.py new file mode 100755 index 0000000000..7126502554 --- /dev/null +++ b/.circleci/test_runner.py @@ -0,0 +1,104 @@ +#!/usr/local/bin/python3 +import logging +import os +import sys +import time + +import requests + + +def create_job(test_runner: str) -> str: + ''' Create a job in the test runner. Returns the job ID + ''' + payload = { + "job": { + "command": "npm", + "args": ["run", "ci-test-fast"], + "cwd": "/home/omg/omg-js" + } + } + try: + request = requests.post(test_runner + '/job', json=payload) + except ConnectionError: + logging.critical('Could not connect to the test runner') + sys.exit(1) # Return a non-zero exit code so CircleCI fails + + logging.info('Job created: {}'.format( + request.content.decode('utf-8')) + ) + return request.content.decode('utf-8') + + +def check_job_completed(test_runner: str, job_id: str): + ''' Get the status of the job from the test runner + ''' + start_time = int(time.time()) + while True: + if start_time >= (start_time + 360): + logging.critical('Test runner did not complete within six minutes') + sys.exit(1) # Return a non-zero exit code so CircleCI fails + try: + request = requests.get( + '{}/job/{}/status'.format(test_runner, job_id), + headers={'Cache-Control': 'no-cache'} + ) + except ConnectionError: + logging.critical('Could not connect to the test runner') + sys.exit(1) # Return a non-zero exit code so CircleCI fails + if 'Exited' in request.content.decode('utf-8'): + logging.info('Job completed successfully') + break + + +def check_job_result(test_runner: str, job_id: str): + ''' Check the result of the job. This is the result of the tests that are + executed against the push. If they all pass 'true' is returned. + ''' + try: + request = requests.get( + test_runner + '/job/{}/success'.format(job_id), + headers={'Cache-Control': 'no-cache'} + ) + except ConnectionError: + logging.critical('Could not connect to the test runner') + sys.exit(1) + if 'true' in request.content.decode('utf-8'): + logging.info('Tests completed successfully') + + +def get_envs() -> dict: + ''' Get the environment variables for the workflow + ''' + envs = {} + test_runner = os.getenv('TEST_RUNNER_SERVICE') + if test_runner is None: + logging.critical('Test runner service ENV missing') + sys.exit(1) # Return a non-zero exit code so CircleCI fails + + envs['TEST_RUNNER_SERVICE'] = test_runner + return envs + + +def start_workflow(): + ''' Get the party started + ''' + logging.info('Workflow started') + envs = get_envs() + job_id = str(create_job(envs['TEST_RUNNER_SERVICE'])) + check_job_completed(envs['TEST_RUNNER_SERVICE'], job_id) + check_job_result(envs['TEST_RUNNER_SERVICE'], job_id) + + +def set_logger(): + ''' Sets the logging module parameters + ''' + root = logging.getLogger('') + for handler in root.handlers: + root.removeHandler(handler) + format = '%(asctime)s %(levelname)-8s:%(message)s' + logging.basicConfig(format=format, level='INFO') + + +if __name__ == '__main__': + set_logger() + start_workflow() From 2a8486b27f2264b66c2b10d58d92ea46456e9ae8 Mon Sep 17 00:00:00 2001 From: jbunce Date: Wed, 12 Jun 2019 14:40:13 +0700 Subject: [PATCH 12/16] fix: Change Python path for Linux systems --- .circleci/test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/test_runner.py b/.circleci/test_runner.py index 7126502554..cc919218fc 100755 --- a/.circleci/test_runner.py +++ b/.circleci/test_runner.py @@ -1,4 +1,4 @@ -#!/usr/local/bin/python3 +#!/usr/bin/python3 import logging import os import sys From f128647181ebbbf9f284805336222058f555eedb Mon Sep 17 00:00:00 2001 From: jbunce Date: Wed, 12 Jun 2019 14:55:04 +0700 Subject: [PATCH 13/16] fix: Add python3-requests lib --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 961a76f576..abe84cca1a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -322,6 +322,7 @@ jobs: - run: name: Functional Tests command: | + apt-get update && apt-get install python3-requests -y python3 .circleci/test_runner.py build_and_deploy_staging: From 15fde55846533afb119488e23674de8f9576f80a Mon Sep 17 00:00:00 2001 From: jbunce Date: Wed, 12 Jun 2019 16:33:41 +0700 Subject: [PATCH 14/16] fix: f you bash --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index abe84cca1a..fa54f63f45 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -318,7 +318,7 @@ jobs: kubectl set image statefulset childchain-samrong childchain=$DOCKER_IMAGE while true; do if [ "$(kubectl get pods childchain-samrong-0 -o jsonpath=\"{.status.phase}\" | grep Running)" ]; then break; fi; done kubectl set image statefulset watcher-samrong watcher=$DOCKER_IMAGE - sleep(120) # TODO: Get the readiness status from each statefulset + sleep 120 # TODO: Get the readiness status from each statefulset - run: name: Functional Tests command: | From b9345c8be341b193298c9c6a68874d7224b2d05a Mon Sep 17 00:00:00 2001 From: jbunce Date: Wed, 12 Jun 2019 17:47:03 +0700 Subject: [PATCH 15/16] fix: Sleep time increase to allow Watcher to deploy --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fa54f63f45..56e5c97f6f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -318,7 +318,7 @@ jobs: kubectl set image statefulset childchain-samrong childchain=$DOCKER_IMAGE while true; do if [ "$(kubectl get pods childchain-samrong-0 -o jsonpath=\"{.status.phase}\" | grep Running)" ]; then break; fi; done kubectl set image statefulset watcher-samrong watcher=$DOCKER_IMAGE - sleep 120 # TODO: Get the readiness status from each statefulset + sleep 200 # TODO: Get the readiness status from each statefulset - run: name: Functional Tests command: | From 34f8dd9426642cdf08c014ef2ba4b9f8c72dc860 Mon Sep 17 00:00:00 2001 From: jbunce Date: Wed, 12 Jun 2019 18:38:20 +0700 Subject: [PATCH 16/16] fix: Appease Murkops --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 56e5c97f6f..a21aee91ab 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -318,7 +318,7 @@ jobs: kubectl set image statefulset childchain-samrong childchain=$DOCKER_IMAGE while true; do if [ "$(kubectl get pods childchain-samrong-0 -o jsonpath=\"{.status.phase}\" | grep Running)" ]; then break; fi; done kubectl set image statefulset watcher-samrong watcher=$DOCKER_IMAGE - sleep 200 # TODO: Get the readiness status from each statefulset + while true; do if [ "$(kubectl get pods watcher-samrong-0 | grep 1/1)" ]; then break; fi; done - run: name: Functional Tests command: |