childchain v2 compatibility with watchers (#1800)

* publish v2 watchers * bump rocksdb for mac * bump rocksdb for mac * old childchain to build the image * old childchain to build the image * disable fees server for old childchain * gcloud login * docker update * pg healthcheck * nginx feefeed healthcheck * bump rocksdb for mac * feefeed geth url * daemon * point to the right specs * exit id size 168 * exit started change * exit started wait * bump git modules, fix tests * disable health checks for chch * inputs txs are not needed in the eth flow * cleanup * cleanup * cleanup * cleanup infura submission tests * cleanup tests for calldata * exit data decoding * block nonce * block nonce * standard exit struct and warnings cleanup * lint fixes, move to specs v2 * slow ife fixes * bump ex plasma * api changes * get correct bond size * refactor test execution * correct network * correct path * get deps and enable docker env var * get deps and enable docker env var * get deps and enable docker env var * get deps and enable docker env var * run all tests through docker container * give the steps names from tests * name2test_name * cache deps * mkdir once * ife deleted event * use specs branch * mapping updates * ex plasma 160 bits * exit deleted log * exit deleted log * aggresive caching * ex plasma 160 bits * ex txhash * tx hash * config url * wait for link * eip comparison * perf setup fees * tx index tx hash from childchain * update perf tests * update perf tests perf setup and cabbage cleanup cleanup cleanup * set fee amount * update fees config from env * perf lint and cleanup * test_url to test_command, git specs branch Co-authored-by: ino.murko@outlook.com <ino.murko@outlook.com> Co-authored-by: Ino Murko <ino@omg.network>
omgnetwork · Feb 10, 2021 · d43f488 · d43f488
1 parent b2ccba7
commit d43f488
Show file tree

Hide file tree

Showing 150 changed files with 1,075 additions and 1,622 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -74,6 +74,12 @@ commands:
     steps:
       - run: printf "%s\\n" "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin
 
+  gcloud_login:
+    steps:
+      - run: |
+          echo $GCLOUD_SERVICE_KEY | gcloud auth activate-service-account --key-file=-
+          gcloud auth configure-docker --quiet
+
   make_docker_images:
     description: Builds docker images
     steps:
@@ -152,6 +158,19 @@ commands:
       - restore_cache:
           key: v2-mix-specs-cache-{{ .Branch }}-{{ checksum "mix.lock" }}
 
+  run_test_in_docker:
+    description: "Quick test runs"
+    parameters:
+      test_command:
+        type: string
+      test_name:
+        type: string
+    steps:
+      - run:
+          name: "Docker run <<parameters.test_name>>"
+          command: |
+            docker run --rm -it --network=project_chain_net -e DOCKER=true -e CHILD_CHAIN_URL=http://172.27.0.108:9656/v1 -e ETHEREUM_RPC_URL=http://172.27.0.108:80 -e DOCKER_GETH=true -e TEST_DATABASE_URL=postgresql://omisego_dev:omisego_dev@172.27.0.107:5432/omisego_test -e SHELL=/bin/sh -v $(pwd):/app --entrypoint /bin/sh omisegoimages/elixir-omg-builder:stable-20201207 -c "cd /app && mix deps.get && <<parameters.test_command>>"
+
   install_deps:
     description: Install linux dependencies
     steps:
@@ -238,11 +257,9 @@ jobs:
           paths:
             - "deps_docker/"
             - "deps_docker/rocksdb"
-            - "_build_docker/test/lib/rocksdb/"
-            - "_build_docker/test/dev/rocksdb/"
+            - "_build_docker/"
             - "deps/"
-            - "_build/test/lib/rocksdb/"
-            - "_build/test/dev/rocksdb/"
+            - "_build/"
       - persist_to_workspace:
           name: Persist workspace
           root: ~/src
@@ -298,7 +315,7 @@ jobs:
             _counter=$(mix credo --only Credo.Check.Readability.SinglePipe | grep -c "Use a function call when a pipeline is only one function long")
             echo "Current Credo.Check.Readability.SinglePipe occurrences:"
             echo $_counter
-            if [ $_counter -gt 289 ]; then
+            if [ $_counter -gt 271 ]; then
               echo "Have you been naughty or nice? Find out if Santa knows."
               exit 1
             fi
@@ -374,94 +391,67 @@ jobs:
       MIX_ENV: test
     steps:
       - checkout
-      - setup_childchain
-      - install_elixir
-      - install_deps
+      - restore_cache:
+          keys:
+            - v2-mix-cache-test-compile-watcher_mix_based_childchain-{{ checksum "mix.lock" }}-{{ .Branch }}
+      - run: rm -rf _build_docker/test/lib/omg*
       - run:
-          name: Compile
-          environment:
-            MIX_ENV: test
+          name: Setup dirs
           command: |
-            set -e
-            make deps-elixir-omg
-            mix compile
-          no_output_timeout: 2400
+            [ -d _build_docker ] || mkdir _build_docker && sudo chmod -R 777 _build_docker
       - run:
-          name: invalid_exit_1_test.exs
-          environment:
-            DOCKER_GETH: true
+          name: Setup dirs deps_docker
           command: |
-            mix test test/omg_watcher/integration/invalid_exit_1_test.exs --include mix_based_child_chain
+            [ -d deps_docker ] || mkdir deps_docker &&  sudo chmod -R 777 deps_docker
       - setup_childchain
-      - run:
-          name: invalid_exit_2_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/invalid_exit_2_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/invalid_exit_1_test.exs --include mix_based_child_chain"
+            test_name: "invalid_exit_1_test.exs"
+      - save_cache:
+          key: v2-mix-cache-test-compile-watcher_mix_based_childchain-{{ checksum "mix.lock" }}-{{ .Branch }}
+          paths:
+            - deps_docker
+            - _build_docker
       - setup_childchain
-      - run:
-          name: block_getter_1_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/block_getter_1_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/in_flight_exit_test_3_test.exs --include mix_based_child_chain"
+            test_name: "in_flight_exit_test_3_test.exs"
       - setup_childchain
-      - run:
-          name: block_getter_2_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/block_getter_2_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/in_flight_exit_test_2_test.exs --include mix_based_child_chain"
+            test_name: "in_flight_exit_test_2_test.exs"
       - setup_childchain
-      - run:
-          name: block_getter_3_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/block_getter_3_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/in_flight_exit_test_1_test.exs --include mix_based_child_chain"
+            test_name: "in_flight_exit_test_1_test.exs"
       - setup_childchain
-      - run:
-          name: block_getter_4_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/block_getter_4_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/in_flight_exit_test_4_test.exs --include mix_based_child_chain"
+            test_name: "in_flight_exit_test_4_test.exs"
       - setup_childchain
-      - run:
-          name: in_flight_exit_test_1_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/in_flight_exit_test_1_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/in_flight_exit_test.exs --include mix_based_child_chain"
+            test_name: "in_flight_exit_test.exs"
       - setup_childchain
-      - run:
-          name: in_flight_exit_test_2_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/in_flight_exit_test_2_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/invalid_exit_2_test.exs --include mix_based_child_chain"
+            test_name: "invalid_exit_2_test.exs"
       - setup_childchain
-      - run:
-          name: in_flight_exit_test_3_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/in_flight_exit_test_3_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/block_getter_1_test.exs --include mix_based_child_chain"
+            test_name: "block_getter_1_test.exs"
       - setup_childchain
-      - run:
-          name: in_flight_exit_test_4_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/in_flight_exit_test_4_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/block_getter_2_test.exs --include mix_based_child_chain"
+            test_name: "block_getter_2_test.exs"
       - setup_childchain
-      - run:
-          name: in_flight_exit_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/in_flight_exit_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/block_getter_3_test.exs --include mix_based_child_chain"
+            test_name: "block_getter_3_test.exs"
+      - setup_childchain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/block_getter_4_test.exs --include mix_based_child_chain"
+            test_name: "block_getter_4_test.exs"
 
   watcher_info_coveralls_and_integration_tests:
     executor: builder_pg_geth
@@ -631,13 +621,15 @@ jobs:
           command: |
             [ -d data ] || mkdir data && chmod 777 data
       - docker_login
+      - gcloud_login
+      - run: pip3 install docker-compose --upgrade
       - make_docker_images
       - run:
           name: Start daemon services
           command: |
             make init_test
             cp ./localchain_contract_addresses.env ./priv/cabbage/apps/itest/localchain_contract_addresses.env
-            docker-compose -f docker-compose.yml -f docker-compose.feefeed.yml -f docker-compose.specs.yml up -d || (START_RESULT=$?; docker-compose logs; exit $START_RESULT;)
+            docker-compose -f docker-compose.yml -f docker-compose.specs.yml up -d || (START_RESULT=$?; docker-compose logs; exit $START_RESULT;)
       - run:
           name: Log daemon services
           command: make cabbage-logs
@@ -647,6 +639,12 @@ jobs:
       - run: sh .circleci/status.sh
       - run:
           name: Run specs
+          environment:
+            CHILD_CHAIN_URL: http://127.0.0.1:9656/v1
+            EXIT_ID_SIZE: "168"
+            LOCALCHAIN_CONTRACT_ADDRESSES: "/home/circleci/project/localchain_contract_addresses.env"
+            PLASMA_CONTRACTS_DIR: "/home/circleci/project/data/plasma-contracts/contracts/"
+            FEE_CLAIMER_ADDRESS: "0x24F3402Cd22F03ff81B56941fE048AD0F4EED5A1"
           command: |
             cd priv/cabbage
             make install
@@ -664,6 +662,8 @@ jobs:
     environment:
       PERF_IMAGE_NAME: "omisego/perf:latest"
       STATIX_TAG: "env:perf_circleci"
+      CHILD_CHAIN_URL: http://127.0.0.1:9656/v1
+      FEE_AMOUNT: 1
     steps:
       - checkout
       - run:
@@ -699,13 +699,13 @@ jobs:
           name: Show help information
           command: docker run -it $PERF_IMAGE_NAME mix run -e "LoadTest.TestRunner.run()" -- help
       - run:
-          name: Run perf smoke test (deposits)
+          name: Run perf smoke test (transactions)
           command: |
-            docker run -it --env-file ./localchain_contract_addresses.env --env DD_API_KEY --env DD_APP_KEY --env STATIX_TAG --network host $PERF_IMAGE_NAME mix run -e "LoadTest.TestRunner.run()" -- "deposits" 1 200
+            docker run -it --env-file ./localchain_contract_addresses.env -e CHILD_CHAIN_URL=http://localhost:9656/v1 -e FEE_AMOUNT=1 --env DD_API_KEY --env DD_APP_KEY --env STATIX_TAG --network host $PERF_IMAGE_NAME mix run -e "LoadTest.TestRunner.run()" -- "transactions" 1 200
       - run:
-          name: Run perf smoke test (transactions)
+          name: Run perf smoke test (deposits)
           command: |
-            docker run -it --env-file ./localchain_contract_addresses.env --env DD_API_KEY --env DD_APP_KEY --env STATIX_TAG --network host $PERF_IMAGE_NAME mix run -e "LoadTest.TestRunner.run()" -- "transactions" 1 200
+            docker run -it --env-file ./localchain_contract_addresses.env -e CHILD_CHAIN_URL=http://localhost:9656/v1 -e FEE_AMOUNT=1 --env DD_API_KEY --env DD_APP_KEY --env STATIX_TAG --network host $PERF_IMAGE_NAME mix run -e "LoadTest.TestRunner.run()" -- "deposits" 1 200
       - run:
           name: (Perf) Format generated code and check for warnings
           command: |
@@ -719,7 +719,6 @@ jobs:
           key: v2-mix-specs-cache-{{ .Branch }}-{{ checksum "mix.lock" }}
           paths:
             - "priv/perf/deps"
-
       - run:
           name: (Perf) Credo and formatting
           command: |
@@ -751,7 +750,7 @@ jobs:
           command: |
             make init_test_reorg
             cp ./localchain_contract_addresses.env ./priv/cabbage/apps/itest/localchain_contract_addresses.env
-            docker-compose -f docker-compose.yml -f docker-compose.feefeed.yml -f docker-compose.reorg.yml -f docker-compose.specs.yml up -d || (START_RESULT=$?; docker-compose logs; exit $START_RESULT;)
+            docker-compose -f docker-compose.yml -f docker-compose.reorg.yml -f docker-compose.specs.yml up -d || (START_RESULT=$?; docker-compose logs; exit $START_RESULT;)
       - run:
           name: Log daemon services
           command: make cabbage-logs-reorg
@@ -781,6 +780,8 @@ jobs:
           background: true
       - run:
           name: Run specs
+          environment:
+            EXIT_ID_SIZE: "168"
           command: |
             cd priv/cabbage
             make install
@@ -882,7 +883,7 @@ jobs:
     machine:
       image: ubuntu-2004:202010-01
     environment:
-      WATCHER_IMAGE_NAME: "omisego/watcher"
+      WATCHER_IMAGE_NAME: "omisego/watcher-v2"
     steps:
       - checkout
       - run: make docker-watcher WATCHER_IMAGE_NAME=$WATCHER_IMAGE_NAME
@@ -910,7 +911,7 @@ jobs:
     machine:
       image: ubuntu-2004:202010-01
     environment:
-      WATCHER_INFO_IMAGE_NAME: "omisego/watcher_info"
+      WATCHER_INFO_IMAGE_NAME: "omisego/watcher_info-v2"
     steps:
       - checkout
       - run: make docker-watcher_info WATCHER_INFO_IMAGE_NAME=$WATCHER_INFO_IMAGE_NAME

diff --git a/Makefile b/Makefile
@@ -79,8 +79,8 @@ list:
 
 all: clean build-watcher-prod build-watcher_info-prod
 
-WATCHER_IMAGE_NAME      ?= "omisego/watcher:latest"
-WATCHER_INFO_IMAGE_NAME ?= "omisego/watcher_info:latest"
+WATCHER_IMAGE_NAME      ?= "omisego/watcher-v2:latest"
+WATCHER_INFO_IMAGE_NAME ?= "omisego/watcher_info-v2:latest"
 
 IMAGE_BUILDER   ?= "omisegoimages/elixir-omg-builder:stable-20201207"
 IMAGE_BUILD_DIR ?= $(PWD)
@@ -190,19 +190,19 @@ init-contracts: clean-contracts
 	cd data && \
 	tar --strip-components 1 -zxvf snapshot.tar.gz data/geth && \
 	tar --exclude=data/* -xvzf snapshot.tar.gz && \
-	AUTHORITY_ADDRESS=$$(cat plasma-contracts/build/authority_address)  \
-	ETH_VAULT=$$(cat plasma-contracts/build/eth_vault)  \
-	ERC20_VAULT=$$(cat plasma-contracts/build/erc20_vault)  \
-	PAYMENT_EXIT_GAME=$$(cat plasma-contracts/build/payment_exit_game)  \
-	PLASMA_FRAMEWORK_TX_HASH=$$(cat plasma-contracts/build/plasma_framework_tx_hash)  \
-	PLASMA_FRAMEWORK=$$(cat plasma-contracts/build/plasma_framework)  \
-	PAYMENT_EIP712_LIBMOCK=$$(cat plasma-contracts/build/paymentEip712LibMock)  \
-	MERKLE_WRAPPER=$$(cat plasma-contracts/build/merkleWrapper)  \
-	ERC20_MINTABLE=$$(cat plasma-contracts/build/erc20Mintable)  \
+	AUTHORITY_ADDRESS=$$(cat plasma-contracts/build/authority_address) && \
+	ETH_VAULT=$$(cat plasma-contracts/build/eth_vault) && \
+	ERC20_VAULT=$$(cat plasma-contracts/build/erc20_vault) && \
+	PAYMENT_EXIT_GAME=$$(cat plasma-contracts/build/payment_exit_game) && \
+	PLASMA_FRAMEWORK_TX_HASH=$$(cat plasma-contracts/build/plasma_framework_tx_hash) && \
+	PLASMA_FRAMEWORK=$$(cat plasma-contracts/build/plasma_framework) && \
+	PAYMENT_EIP712_LIBMOCK=$$(cat plasma-contracts/build/paymentEip712LibMock) && \
+	MERKLE_WRAPPER=$$(cat plasma-contracts/build/merkleWrapper) && \
+	ERC20_MINTABLE=$$(cat plasma-contracts/build/erc20Mintable) && \
+	sh ../bin/generate-localchain-env AUTHORITY_ADDRESS=$$AUTHORITY_ADDRESS ETH_VAULT=$$ETH_VAULT \
 	ERC20_VAULT=$$ERC20_VAULT PAYMENT_EXIT_GAME=$$PAYMENT_EXIT_GAME \
 	PLASMA_FRAMEWORK_TX_HASH=$$PLASMA_FRAMEWORK_TX_HASH PLASMA_FRAMEWORK=$$PLASMA_FRAMEWORK \
-	PAYMENT_EIP712_LIBMOCK=$$PAYMENT_EIP712_LIBMOCK MERKLE_WRAPPER=$$MERKLE_WRAPPER ERC20_MINTABLE=$$ERC20_MINTABLE \
-	sh ../bin/generate-localchain-env
+	PAYMENT_EIP712_LIBMOCK=$$PAYMENT_EIP712_LIBMOCK MERKLE_WRAPPER=$$MERKLE_WRAPPER ERC20_MINTABLE=$$ERC20_MINTABLE
 
 
 init-contracts-reorg: clean-contracts
@@ -398,7 +398,7 @@ docker-remote-watcher_info:
 ###
 start-services:
 	SNAPSHOT=SNAPSHOT_MIX_EXIT_PERIOD_SECONDS_120 make init_test && \
-	docker-compose -f ./docker-compose.yml -f ./docker-compose.feefeed.yml up feefeed geth nginx postgres
+	docker-compose -f ./docker-compose.yml up feefeed geth nginx postgres
 
 start-watcher:
 	. ${OVERRIDING_VARIABLES} && \

diff --git a/apps/omg/lib/omg/state/core.ex b/apps/omg/lib/omg/state/core.ex
@@ -369,9 +369,9 @@ defmodule OMG.State.Core do
       Gets the current block's height and whether at the beginning of the block
   """
   @spec get_status(t()) :: {current_block_height :: non_neg_integer(), is_block_beginning :: boolean()}
-  def get_status(%__MODULE__{height: height, tx_index: tx_index, pending_txs: pending}) do
-    is_beginning = tx_index == 0 && Enum.empty?(pending)
-    {height, is_beginning}
+  def get_status(state) do
+    is_beginning = state.tx_index == 0 && Enum.empty?(state.pending_txs)
+    {state.height, is_beginning}
   end
 
   defp add_pending_tx(%Core{pending_txs: pending_txs, tx_index: tx_index} = state, %Transaction.Recovered{} = new_tx) do

diff --git a/apps/omg/lib/omg/state/transaction/fee.ex b/apps/omg/lib/omg/state/transaction/fee.ex
@@ -153,8 +153,9 @@ defimpl OMG.State.Transaction.Protocol, for: OMG.State.Transaction.Fee do
          do: {:ok, %{}}
   end
 
-  defp find_output_by_currency(outputs, currency),
-    do: Enum.find(outputs, {:error, :surplus_in_token_not_collected}, fn o -> o.currency == currency end)
+  defp find_output_by_currency(outputs, currency) do
+    Enum.find(outputs, {:error, :surplus_in_token_not_collected}, fn o -> o.currency == currency end)
+  end
 
   defp amounts_equal?(collected, claimed) when collected == claimed, do: true
   defp amounts_equal?(_, _), do: {:error, :claimed_and_collected_amounts_mismatch}

diff --git a/apps/omg/mix.exs b/apps/omg/mix.exs
@@ -40,7 +40,7 @@ defmodule OMG.MixProject do
 
   defp deps() do
     [
-      {:ex_plasma, git: "https://github.com/omgnetwork/ex_plasma.git", ref: "867fee97dd930b47d22b5b32c4ef1751424c4206"},
+      {:ex_plasma, "~> 0.3.0"},
       {:ex_rlp, "~> 0.5.3"},
       {:merkle_tree, "~> 2.0.0"},
       {:telemetry, "~> 0.4.1"},

diff --git a/apps/omg/test/fixtures.exs b/apps/omg/test/fixtures.exs
@@ -19,7 +19,7 @@ defmodule OMG.Fixtures do
   alias OMG.Eth.Configuration
   alias OMG.State.Core
 
-  @eth OMG.Eth.zero_address()
+  @eth <<0::160>>
   @fee_claimer_address "NO FEE CLAIMER ADDR!"
 
   deffixture(entities, do: entities())