get transaction api batch (#1816)

* get transaction api batch * deplyo chch 1 changes to development * dialyzer * passthrough the txbytes * dialyzer * speed up tests * point cabbage to master * speed up tests * speed up tests - fix nginx * port cleanups, remove nginx from root * harden test, silent geth * harden test, silent geth * encoding fix * route to chch * typo * docker mock feefeed * disable test_docker_compose_performance
omgnetwork · Feb 25, 2021 · ce34cfa · ce34cfa
1 parent fbed2fc
commit ce34cfa
Show file tree

Hide file tree

Showing 41 changed files with 666 additions and 221 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -126,6 +126,19 @@ commands:
               sleep 5
             done
 
+  run_test_in_docker:
+    description: "Quick test runs"
+    parameters:
+      test_command:
+        type: string
+      test_name:
+        type: string
+    steps:
+      - run:
+          name: "Docker run <<parameters.test_name>>"
+          command: |
+            docker run --rm -it --network=project_chain_net -e DOCKER=true -e CHILD_CHAIN_URL=http://172.27.0.108:9656/ -e ETHEREUM_RPC_URL=http://172.27.0.108:80 -e DOCKER_GETH=true -e TEST_DATABASE_URL=postgresql://omisego_dev:omisego_dev@172.27.0.107:5432/omisego_test -e SHELL=/bin/sh -v $(pwd):/app --entrypoint /bin/sh omisegoimages/elixir-omg-builder:stable-20201207 -c "cd /app && mix deps.get && <<parameters.test_command>>"
+
   install_elixir:
     description: Installs elixir and checks if docker is healthy
     steps:
@@ -312,7 +325,7 @@ jobs:
             _counter=$(mix credo --only Credo.Check.Readability.SinglePipe | grep -c "Use a function call when a pipeline is only one function long")
             echo "Current Credo.Check.Readability.SinglePipe occurrences:"
             echo $_counter
-            if [ $_counter -gt 289 ]; then
+            if [ $_counter -gt 283 ]; then
               echo "Have you been naughty or nice? Find out if Santa knows."
               exit 1
             fi
@@ -367,94 +380,67 @@ jobs:
       MIX_ENV: test
     steps:
       - checkout
-      - setup_childchain
-      - install_elixir
-      - install_deps
+      - restore_cache:
+          keys:
+            - v2-mix-cache-test-compile-watcher_mix_based_childchain-{{ checksum "mix.lock" }}-{{ .Branch }}
+      - run: rm -rf _build_docker/test/lib/omg*
       - run:
-          name: Compile
-          environment:
-            MIX_ENV: test
+          name: Setup dirs
           command: |
-            set -e
-            make deps-elixir-omg
-            mix compile
-          no_output_timeout: 2400
+            [ -d _build_docker ] || mkdir _build_docker && sudo chmod -R 777 _build_docker
       - run:
-          name: invalid_exit_1_test.exs
-          environment:
-            DOCKER_GETH: true
+          name: Setup dirs deps_docker
           command: |
-            mix test test/omg_watcher/integration/invalid_exit_1_test.exs --include mix_based_child_chain
+            [ -d deps_docker ] || mkdir deps_docker &&  sudo chmod -R 777 deps_docker
       - setup_childchain
-      - run:
-          name: invalid_exit_2_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/invalid_exit_2_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/invalid_exit_1_test.exs --include mix_based_child_chain"
+            test_name: "invalid_exit_1_test.exs"
+      - save_cache:
+          key: v2-mix-cache-test-compile-watcher_mix_based_childchain-{{ checksum "mix.lock" }}-{{ .Branch }}
+          paths:
+            - deps_docker
+            - _build_docker
       - setup_childchain
-      - run:
-          name: block_getter_1_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/block_getter_1_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/in_flight_exit_test_3_test.exs --include mix_based_child_chain"
+            test_name: "in_flight_exit_test_3_test.exs"
       - setup_childchain
-      - run:
-          name: block_getter_2_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/block_getter_2_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/in_flight_exit_test_2_test.exs --include mix_based_child_chain"
+            test_name: "in_flight_exit_test_2_test.exs"
       - setup_childchain
-      - run:
-          name: block_getter_3_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/block_getter_3_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/in_flight_exit_test_1_test.exs --include mix_based_child_chain"
+            test_name: "in_flight_exit_test_1_test.exs"
       - setup_childchain
-      - run:
-          name: block_getter_4_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/block_getter_4_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/in_flight_exit_test_4_test.exs --include mix_based_child_chain"
+            test_name: "in_flight_exit_test_4_test.exs"
       - setup_childchain
-      - run:
-          name: in_flight_exit_test_1_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/in_flight_exit_test_1_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/in_flight_exit_test.exs --include mix_based_child_chain"
+            test_name: "in_flight_exit_test.exs"
       - setup_childchain
-      - run:
-          name: in_flight_exit_test_2_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/in_flight_exit_test_2_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/invalid_exit_2_test.exs --include mix_based_child_chain"
+            test_name: "invalid_exit_2_test.exs"
       - setup_childchain
-      - run:
-          name: in_flight_exit_test_3_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/in_flight_exit_test_3_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/block_getter_1_test.exs --include mix_based_child_chain"
+            test_name: "block_getter_1_test.exs"
       - setup_childchain
-      - run:
-          name: in_flight_exit_test_4_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/in_flight_exit_test_4_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/block_getter_2_test.exs --include mix_based_child_chain"
+            test_name: "block_getter_2_test.exs"
       - setup_childchain
-      - run:
-          name: in_flight_exit_test.exs
-          environment:
-            DOCKER_GETH: true
-          command: |
-            mix test test/omg_watcher/integration/in_flight_exit_test.exs --include mix_based_child_chain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/block_getter_3_test.exs --include mix_based_child_chain"
+            test_name: "block_getter_3_test.exs"
+      - setup_childchain
+      - run_test_in_docker:
+            test_command: "mix test test/omg_watcher/integration/block_getter_4_test.exs --include mix_based_child_chain"
+            test_name: "block_getter_4_test.exs"
 
   watcher_info_coveralls_and_integration_tests:
     executor: builder_pg_geth
@@ -630,10 +616,7 @@ jobs:
       - install_and_setup_gcloud
       - run:
           name: Start daemon services
-          command: |
-            make init_test
-            cp ./localchain_contract_addresses.env ./priv/cabbage/apps/itest/localchain_contract_addresses.env
-            docker-compose -f docker-compose.yml -f docker-compose.feefeed.yml -f docker-compose.specs.yml up -d || (START_RESULT=$?; docker-compose logs; exit $START_RESULT;)
+          command: make cabbage-start-services
       - run:
           name: Log daemon services
           command: make cabbage-logs
@@ -648,10 +631,22 @@ jobs:
             make install
             make generate_api_code
             mix deps.get
-            grep -r "@moduletag" ./apps/itest/ | awk '{print "--include " $3}' | tr -d ':' > tags.txt
-            echo $(circleci tests split ./tags.txt)
-            circleci tests split ./tags.txt > /tmp/tests-to-run
-            mix test --exclude test $(cat /tmp/tests-to-run)
+      - run:
+          name: Run specs
+          working_directory: ~/project/priv/cabbage
+          environment:
+            MIX_ENV: test
+          command: |
+            mix compile
+      - run:
+          name: Run specs
+          working_directory: ~/project/priv/cabbage
+          command: |
+            TESTFILES=$(circleci tests glob "apps/itest/test/itest/*_test.exs" | circleci tests split --split-by=timings --show-counts)
+            echo ${TESTFILES}
+            mix test ${TESTFILES} --trace
+      - store_test_results:
+          path: ~/project/priv/cabbage/_build/test/lib/itest/
 
   test_docker_compose_performance:
     description: "These are not actually performance tests, we're checking if the scripts work"
@@ -700,11 +695,11 @@ jobs:
       - run:
           name: Run perf smoke test (deposits)
           command: |
-            docker run -it --env-file ./localchain_contract_addresses.env --env DD_API_KEY --env DD_APP_KEY --env STATIX_TAG --network host $PERF_IMAGE_NAME mix run -e "LoadTest.TestRunner.run()" -- "deposits" 1 200
+            docker run -it --env-file ./localchain_contract_addresses.env -e FEE_AMOUNT=1 --env DD_API_KEY --env DD_APP_KEY --env STATIX_TAG --network host $PERF_IMAGE_NAME mix run -e "LoadTest.TestRunner.run()" -- "deposits" 1 200
       - run:
           name: Run perf smoke test (transactions)
           command: |
-            docker run -it --env-file ./localchain_contract_addresses.env --env DD_API_KEY --env DD_APP_KEY --env STATIX_TAG --network host $PERF_IMAGE_NAME mix run -e "LoadTest.TestRunner.run()" -- "transactions" 1 200
+            docker run -it --env-file ./localchain_contract_addresses.env -e FEE_AMOUNT=1 --env DD_API_KEY --env DD_APP_KEY --env STATIX_TAG --network host $PERF_IMAGE_NAME mix run -e "LoadTest.TestRunner.run()" -- "transactions" 1 200
       - run:
           name: (Perf) Format generated code and check for warnings
           command: |
@@ -1044,7 +1039,7 @@ workflows:
       - integration_tests:
           requires: [build]
       - barebuild_macos
-      - test_barebone_release
+      #- test_barebone_release
   build-test-deploy:
     jobs:
       - build:
@@ -1053,8 +1048,8 @@ workflows:
               only: /.+/
             tags:
               only: /.+/
-      - test_barebone_release:
-          filters: *all_branches_and_tags
+      # - test_barebone_release:
+      #     filters: *all_branches_and_tags
       - notify_services:
           requires:
             - increase_chart_version_watcher_master
@@ -1080,8 +1075,8 @@ workflows:
           filters: *all_branches_and_tags
       - test_docker_compose_release:
           filters: *all_branches_and_tags
-      - test_docker_compose_performance:
-          filters: *all_branches_and_tags
+      # - test_docker_compose_performance:
+      #     filters: *all_branches_and_tags
       - test_docker_compose_reorg:
           filters:
             branches:
@@ -1111,7 +1106,7 @@ workflows:
       - publish_watcher:
           requires:
             [
-              test_barebone_release,
+             # test_barebone_release,
               test_docker_compose_release,
               watcher_coveralls_and_integration_tests,
               watcher_info_coveralls_and_integration_tests,
@@ -1134,7 +1129,7 @@ workflows:
       - publish_watcher_info:
           requires:
             [
-              test_barebone_release,
+             # test_barebone_release,
               test_docker_compose_release,
               watcher_coveralls_and_integration_tests,
               watcher_info_coveralls_and_integration_tests,
@@ -1147,17 +1142,17 @@ workflows:
             ]
           filters: *master_and_version_branches_and_all_tags
 
-      - publish_perf:
-          requires: [test_docker_compose_performance]
-          filters:
-            branches:
-              only:
-                - master
-                # vMAJOR.MINOR (e.g. v0.1, v0.2, v1.0, v2.1, etc.)
-                - /^v[0-9]+\.[0-9]+/
-            tags:
-              only:
-                - /.+/
+      # - publish_perf:
+      #     requires: [test_docker_compose_performance]
+      #     filters:
+      #       branches:
+      #         only:
+      #           - master
+      #           # vMAJOR.MINOR (e.g. v0.1, v0.2, v1.0, v2.1, etc.)
+      #           - /^v[0-9]+\.[0-9]+/
+      #       tags:
+      #         only:
+      #           - /.+/
       # Increase chart version for master, this will end up trigger deployment on dev
       - increase_chart_version_watcher_master:
           requires: [publish_watcher, publish_watcher_info]
@@ -1186,7 +1181,7 @@ workflows:
           filters: *only_release_tag
       - release:
           requires: [
-            test_barebone_release,
+           # test_barebone_release,
             test_docker_compose_release,
             watcher_coveralls_and_integration_tests,
             watcher_info_coveralls_and_integration_tests,

diff --git a/.gitmodules b/.gitmodules
@@ -2,3 +2,4 @@
   path = priv/cabbage
   url = https://github.com/omgnetwork/specs.git
   branch = master
+
diff --git a/Makefile b/Makefile
@@ -342,6 +342,20 @@ cabbage-reorg-geth-logs:
 cabbage-reorgs-logs:
 	docker-compose -f docker-compose.yml -f docker-compose.reorg.yml -f docker-compose.specs.yml logs --follow | grep "reorg"
 
+### Cabbage service commands
+
+cabbage-start-services:
+	make init_test && \
+        cp ./localchain_contract_addresses.env ./priv/cabbage/apps/itest/localchain_contract_addresses.env && \
+	docker-compose -f docker-compose.yml -f docker-compose.specs.yml up -d || \
+	(START_RESULT=$?; docker-compose logs; exit $START_RESULT;)
+
+cabbage-start-services-reorg:
+	make init_test_reorg && \
+	cp ./localchain_contract_addresses.env ./priv/cabbage/apps/itest/localchain_contract_addresses.env && \
+	docker-compose -f docker-compose.yml -f docker-compose.reorg.yml -f docker-compose.specs.yml up -d || \
+	(START_RESULT=$?; docker-compose logs; exit $START_RESULT;)
+
 ###OTHER
 docker-start-cluster:
 	SNAPSHOT=SNAPSHOT_MIX_EXIT_PERIOD_SECONDS_120 make init_test && \
@@ -407,7 +421,7 @@ start-watcher:
 	rm -f ./_build/${BAREBUILD_ENV}/rel/watcher/var/sys.config || true && \
 	echo "Init Watcher DBs" && \
 	_build/${BAREBUILD_ENV}/rel/watcher/bin/watcher eval "OMG.DB.ReleaseTasks.InitKeyValueDB.run()" && \
-	_build/${BAREBUILD_ENV}/rel/watcher_info/bin/watcher_info eval "OMG.DB.ReleaseTasks.InitKeysWithValues.run()" && \
+	_build/${BAREBUILD_ENV}/rel/watcher/bin/watcher eval "OMG.DB.ReleaseTasks.InitKeysWithValues.run()" && \
 	echo "Run Watcher" && \
 	. ${OVERRIDING_VARIABLES} && \
 	PORT=${WATCHER_PORT} _build/${BAREBUILD_ENV}/rel/watcher/bin/watcher $(OVERRIDING_START)

diff --git a/apps/omg_eth/lib/omg_eth/encoding.ex b/apps/omg_eth/lib/omg_eth/encoding.ex
@@ -39,8 +39,8 @@ defmodule OMG.Eth.Encoding do
   """
   # because https://github.com/rrrene/credo/issues/583, we need to:
   # credo:disable-for-next-line Credo.Check.Consistency.SpaceAroundOperators
-  @spec from_hex(<<_::16, _::_*8>>) :: binary
-  def from_hex("0x" <> encoded), do: Base.decode16!(encoded, case: :lower)
+  @spec from_hex(<<_::16, _::_*8>>, atom()) :: binary
+  def from_hex("0x" <> encoded, format \\ :lower), do: Base.decode16!(encoded, case: format)
 
   @doc """
   Decodes to an integer, see `to_hex`

diff --git a/apps/omg_eth/test/fixtures.exs b/apps/omg_eth/test/fixtures.exs
@@ -72,7 +72,7 @@ defmodule OMG.Eth.Fixtures do
 
   defp has_exit_queue(vault_id, token) do
     plasma_framework = Configuration.contracts().plasma_framework
-    token = Encoding.from_hex(token)
+    token = Encoding.from_hex(token, :mixed)
     call_contract(plasma_framework, "hasExitQueue(uint256,address)", [vault_id, token], [:bool])
   end
 

diff --git a/apps/omg_utils/lib/omg_utils/http_rpc/response.ex b/apps/omg_utils/lib/omg_utils/http_rpc/response.ex
@@ -56,7 +56,7 @@ defmodule OMG.Utils.HttpRPC.Response do
   end
 
   def sanitize(list) when is_list(list) do
-    list |> Enum.map(&sanitize/1)
+    Enum.map(list, &sanitize/1)
   end
 
   def sanitize(map_or_struct) when is_map(map_or_struct) do

diff --git a/apps/omg_watcher/lib/omg_watcher/api/transaction.ex b/apps/omg_watcher/lib/omg_watcher/api/transaction.ex
@@ -33,6 +33,12 @@ defmodule OMG.Watcher.API.Transaction do
   * transaction doesn't spend funds not yet mined
   * etc...
   """
+  @spec submit(list(Transaction.Signed.t())) :: Client.response_t() | {:error, atom()}
+  def batch_submit(signed_txs) do
+    url = Application.get_env(:omg_watcher, :child_chain_url)
+    Client.batch_submit(signed_txs, url)
+  end
+
   @spec submit(Transaction.Signed.t()) :: Client.response_t() | {:error, atom()}
   def submit(%Transaction.Signed{} = signed_tx) do
     url = Application.get_env(:omg_watcher, :child_chain_url)