From b0a0d5de82fcb9ad28cde6c27522e82f071ca3f2 Mon Sep 17 00:00:00 2001 From: tenmoves Date: Fri, 3 Mar 2023 13:43:46 +0100 Subject: [PATCH] fix cd pipeline for code proposals --- .github/workflows/ci.yml | 117 +++++++------ .iex.exs | 26 +++ Dockerfile | 21 +-- Makefile | 1 - assets/package-lock.json | 8 +- config/config.exs | 6 +- config/dev.exs | 6 +- config/prod.exs | 6 +- config/test.exs | 6 +- .../bootstrap/transaction_handler.ex | 4 +- lib/archethic/crypto.ex | 36 ++-- lib/archethic/db/embedded_impl/chain_index.ex | 2 +- .../election/hypergeometric_distribution.ex | 10 +- .../governance/code/cicd/docker/cicd.ex | 117 +++++++------ .../governance/code/proposal/validator.ex | 37 +++- .../ip_lookup/nat_discovery/miniupnp.ex | 12 +- lib/archethic/networking/port_forwarding.ex | 2 +- lib/archethic/p2p/listener.ex | 2 +- lib/archethic/utils/regression.ex | 17 +- lib/archethic/utils/testnet.ex | 159 +++++++++--------- .../plugs/plug_throttle_by_ip_and_path.ex | 2 + .../plugs/plug_throttle_by_ip_high.ex | 2 + .../plugs/plug_throttle_by_ip_low.ex | 2 + lib/mix/tasks/validate.ex | 52 ++++++ mix.exs | 6 +- mix.lock | 2 +- package-lock.json | 6 + rel/commands/validate | 3 + rel/config.exs | 5 +- scripts/governance/proposal_ci_job.sh | 4 +- .../archethic/oracle_chain/scheduler_test.exs | 2 - 31 files changed, 408 insertions(+), 273 deletions(-) create mode 100644 lib/mix/tasks/validate.ex create mode 100644 package-lock.json create mode 100755 rel/commands/validate diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8aefa25ae8..b18f0eead0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,72 +2,71 @@ name: Elixir CI on: push: - branches: [ master, develop, testnet ] + branches: [master, develop, testnet] pull_request: - branches: [ master, develop, testnet ] + branches: [master, develop, testnet] env: MIX_ENV: test jobs: build: - name: Build and test runs-on: ubuntu-20.04 - + steps: - - name: Install OS Packages - uses: mstksg/get-package@2a4b48d55d72d43ca89ae58ec9ca1397d34a1c35 - with: - apt-get: libgmp-dev libssl-dev libtinfo-dev libsystemd-dev zlib1g-dev libsodium-dev autoconf-archive libcmocka0 libcmocka-dev procps iproute2 build-essential git pkg-config gcc libtool automake libssl-dev uthash-dev autoconf doxygen libjson-c-dev libini-config-dev libcurl4-openssl-dev libltdl-dev libtss2-dev tss2 - - uses: actions/checkout@v2 - - name: Set up Elixir - uses: erlef/setup-beam@988e02bfe678367a02564f65ca2e37726dc0268f - id: beam - with: - elixir-version: '1.14.1' # Define the elixir version [required] - otp-version: '25.1' # Define the OTP version [required] - - name: Restore dependencies cache - uses: actions/cache@v3 - id: mix-cache - with: - path: deps - key: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-mix-${{ hashFiles('**/mix.lock') }} - restore-keys: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-mix- - - name: Make Clean - run: mix clean - - name: Install dependencies - if: steps.mix-cache.outputs.cache-hit != 'true' - run: mix deps.get - - name: Check dependency updates - run: mix hex.outdated --within-requirements 1>/dev/null || echo 'Updates available!' - - name: Set Formatting - run: mix format --check-formatted - - name: Restore build cache - uses: actions/cache@v3 - with: - path: _build - key: ${{ runner.os }}-build-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('**/mix.lock') }} - restore-keys: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-build- - - name: Compile the codebase - run: mix compile --warnings-as-errors - - name: Set credo - run: mix credo - - name: Run Sobelow - run: mix sobelow - - name: Run tests - run: mix test --trace - - name: Retrieve PLT Cache - uses: actions/cache@v3 - id: plt-cache - with: - path: priv/plts - key: ${{ runner.os }}-plts-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('**/mix.lock') }} - restore-keys: ${{ runner.os }}-plts-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}- - - name: Create PLTs - if: steps.plt-cache.outputs.cache-hit != 'true' - run: | - mkdir -p priv/plts - mix dialyzer --plt - - name: Run dialyzer - run: mix dialyzer --no-check --ignore-exit-status + - name: Install OS Packages + uses: mstksg/get-package@2a4b48d55d72d43ca89ae58ec9ca1397d34a1c35 + with: + apt-get: libgmp-dev libssl-dev libtinfo-dev libsystemd-dev zlib1g-dev libsodium-dev autoconf-archive libcmocka0 libcmocka-dev procps iproute2 build-essential git pkg-config gcc libtool automake libssl-dev uthash-dev autoconf doxygen libjson-c-dev libini-config-dev libcurl4-openssl-dev libltdl-dev libtss2-dev tss2 + - uses: actions/checkout@v2 + - name: Set up Elixir + uses: erlef/setup-beam@988e02bfe678367a02564f65ca2e37726dc0268f + id: beam + with: + elixir-version: "1.14.1" # Define the elixir version [required] + otp-version: "25.1" # Define the OTP version [required] + - name: Restore dependencies cache + uses: actions/cache@v3 + id: mix-cache + with: + path: deps + key: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-mix-${{ hashFiles('**/mix.lock') }} + restore-keys: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-mix- + - name: Make Clean + run: mix clean + - name: Install dependencies + if: steps.mix-cache.outputs.cache-hit != 'true' + run: mix deps.get + - name: Check dependency updates + run: mix hex.outdated --within-requirements 1>/dev/null || echo 'Updates available!' + - name: Set Formatting + run: mix format --check-formatted + - name: Restore build cache + uses: actions/cache@v3 + with: + path: _build + key: ${{ runner.os }}-build-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('**/mix.lock') }} + restore-keys: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-build- + - name: Compile the codebase + run: mix compile --warnings-as-errors + - name: Set credo + run: mix credo + - name: Run Sobelow + run: mix sobelow + - name: Run tests + run: mix test --trace + - name: Retrieve PLT Cache + uses: actions/cache@v3 + id: plt-cache + with: + path: priv/plts + key: ${{ runner.os }}-plts-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('**/mix.lock') }} + restore-keys: ${{ runner.os }}-plts-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}- + - name: Create PLTs + if: steps.plt-cache.outputs.cache-hit != 'true' + run: | + mkdir -p priv/plts + mix dialyzer --plt + - name: Run dialyzer + run: mix dialyzer --no-check --ignore-exit-status diff --git a/.iex.exs b/.iex.exs index 6dbc4a11a2..ba851bcb83 100644 --- a/.iex.exs +++ b/.iex.exs @@ -13,3 +13,29 @@ alias Archethic.TransactionChain alias Archethic.TransactionChain.Transaction alias Archethic.TransactionChain.TransactionData alias Archethic.BeaconChain + + +alias Archethic.Governance.Code.CICD.Docker +alias Archethic.Governance.Code +alias Archethic.Governance +prop = %Archethic.Governance.Code.Proposal{ + address: <<0, 0, 135, 189, 175, 8, 43, 160, 49, 21, 23, 241, 61, 189, 221, + 177, 45, 120, 59, 243, 80, 75, 193, 250, 119, 188, 219, 73, 209, 197, 118, + 170, 122, 142>>, + previous_public_key: <<0, 1, 62, 205, 23, 142, 29, 217, 94, 202, 136, 109, + 165, 135, 153, 205, 202, 132, 122, 116, 97, 18, 90, 34, 141, 225, 172, 138, + 16, 138, 85, 15, 41, 149>>, + timestamp: nil, + description: "\Testing code proposal\"", + changes: "diff --git a/mix.exs b/mix.exs\nindex a82c0b3c..b94d4323 100644\n--- a/mix.exs\n+++ b/mix.exs\n@@ -4,7 +4,7 @@ defmodule Archethic.MixProject do\n def project do\n [\n app: :archethic,\n- version: \"1.0.7\",\n+ version: \"1.0.8\",\n build_path: \"_build\",\n config_path: \"config/config.exs\",\n deps_path: \"deps\",\ndiff --git a/rel/appups/archethic/1.0.7_to_1.0.8.appup b/rel/appups/archethic/1.0.7_to_1.0.8.appup\nnew file mode 100644\nindex 00000000..18b6d541\n--- /dev/null\n+++ b/rel/appups/archethic/1.0.7_to_1.0.8.appup\n@@ -0,0 +1,4 @@\n+{\"1.0.8\",\n+ [{\"1.0.7\",\n+ [{load_module,'Elixir.Archethic', []}]}],\n+ [{\"1.0.7\",\n+ [{load_module,'Elixir.Archethic', []}]}]\n+}.\n", + version: "1.0.8", + files: ["mix.exs", "rel/appups/archethic/1.0.7_to_1.0.8.appup"], + approvals: [ + <<0, 0, 135, 189, 175, 8, 43, 160, 49, 21, 23, 241, 61, 189, 221, 177, 45, + 120, 59, 243, 80, 75, 193, 250, 119, 188, 219, 73, 209, 197, 118, 170, + 122, 142>>, + <<0, 0, 168, 220, 209, 92, 167, 255, 187, 168, 176, 63, 128, 210, 199, 35, + 63, 0, 252, 36, 147, 196, 130, 249, 233, 17, 132, 0, 6, 88, 232, 198, 98, + 44>> + ] +} diff --git a/Dockerfile b/Dockerfile index ef70f11f3b..70cc15ddce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,21 +23,20 @@ ARG MIX_ENV=prod RUN apk add --no-cache --update \ build-base \ + grep \ bash \ gcc \ make \ g++ \ - libexecinfo-dev \ - libexecinfo \ git \ npm \ python3 \ wget \ openssl \ libsodium-dev \ + libexecinfo-dev \ gmp-dev - # Install hex and rebar RUN mix local.rebar --force \ && mix local.hex --if-missing --force @@ -60,23 +59,15 @@ RUN git config user.name aebot \ && git config user.email aebot@archethic.net \ && git remote add origin https://github.com/archethic-foundation/archethic-node -# Install Dart Sass -RUN npm install -g sass - -# build Sass -> CSS -RUN cd assets && \ - sass --no-source-map --style=compressed css/app.scss ../priv/static/css/app.css && cd - - # build release -RUN mix do assets.deploy, distillery.release - +RUN mix assets.deploy +RUN MIX_ENV=prod mix distillery.release # gen PLT RUN if [ $with_tests -eq 1 ]; then mix git_hooks.run pre_push ;fi # Install RUN mkdir -p /opt/app \ - && cd /opt/app \ - && tar zxf /opt/code/_build/${MIX_ENV}/rel/archethic_node/releases/*/archethic_node.tar.gz + && tar zxf /opt/code/_build/${MIX_ENV}/rel/archethic_node/releases/*/archethic_node.tar.gz -C /opt/app CMD /opt/app/bin/archethic_node foreground ################################################################################ @@ -85,7 +76,7 @@ FROM archethic-ci as build FROM elixir:1.14.1-alpine -RUN apk add --no-cache --update bash git openssl libsodium +RUN apk add --no-cache --update bash git openssl libsodium libexecinfo COPY --from=build /opt/app /opt/app COPY --from=build /opt/code/.git /opt/code/.git diff --git a/Makefile b/Makefile index a839a63600..1d153fec40 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,6 @@ ifeq ($(TPM_INSTALLED),0) $(CC) src/c/crypto/tpm/keygen.c src/c/crypto/tpm/lib.c -o priv/c_dist/tpm_keygen -I src/c/crypto/tpm/lib.h $(TPMFLAGS) endif - clean: rm -f priv/c_dist/* mix archethic.clean_db diff --git a/assets/package-lock.json b/assets/package-lock.json index 43d95341e0..54b53e4e2d 100644 --- a/assets/package-lock.json +++ b/assets/package-lock.json @@ -24,13 +24,15 @@ } }, "../deps/phoenix": { - "version": "0.0.1" + "version": "1.6.15", + "license": "MIT" }, "../deps/phoenix_html": { - "version": "0.0.1" + "version": "3.2.0" }, "../deps/phoenix_live_view": { - "version": "0.0.1" + "version": "0.18.11", + "license": "MIT" }, "node_modules/@ampproject/remapping": { "version": "2.2.0", diff --git a/config/config.exs b/config/config.exs index 8c883d538f..e7ca22a517 100644 --- a/config/config.exs +++ b/config/config.exs @@ -9,12 +9,12 @@ config :git_hooks, {:cmd, "mix clean"}, {:cmd, "mix format --check-formatted"}, {:cmd, "mix compile --warnings-as-errors"}, + {:cmd, "mix test --trace"}, {:cmd, "mix credo"}, {:cmd, "mix sobelow"}, {:cmd, "mix knigge.verify"}, - {:cmd, "mix test --trace"}, - {:cmd, "mix dialyzer"}, - {:cmd, "mix check.updates"} + {:cmd, "mix dialyzer"} + # {:cmd, "mix check.updates"} ] ] ] diff --git a/config/dev.exs b/config/dev.exs index f50c59bf7a..2169b49731 100755 --- a/config/dev.exs +++ b/config/dev.exs @@ -167,13 +167,13 @@ config :archethic, ArchethicWeb.Endpoint, config :archethic, :throttle, by_ip_high: [ period: 1000, - limit: 5_000 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_HIGH", "5000") ], by_ip_low: [ period: 1000, - limit: 5_000 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_LOW", "5000") ], by_ip_and_path: [ period: 1000, - limit: 5_000 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_AND_PATH", "5000") ] diff --git a/config/prod.exs b/config/prod.exs index 828205dd41..21f8f697dc 100755 --- a/config/prod.exs +++ b/config/prod.exs @@ -273,13 +273,13 @@ config :archethic, ArchethicWeb.Endpoint, config :archethic, :throttle, by_ip_high: [ period: 1000, - limit: 500 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_HIGH", "500") ], by_ip_low: [ period: 1000, - limit: 20 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_LOW", "20") ], by_ip_and_path: [ period: 1000, - limit: 20 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_AND_PATH", "20") ] diff --git a/config/test.exs b/config/test.exs index 66a0f6fb6c..57dd04f3ad 100755 --- a/config/test.exs +++ b/config/test.exs @@ -176,13 +176,13 @@ config :archethic, ArchethicWeb.Endpoint, config :archethic, :throttle, by_ip_high: [ period: 1000, - limit: 5_000 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_HIGH", "5000") ], by_ip_low: [ period: 1000, - limit: 5_000 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_LOW", "5000") ], by_ip_and_path: [ period: 1000, - limit: 5_000 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_AND_PATH", "5000") ] diff --git a/lib/archethic/bootstrap/transaction_handler.ex b/lib/archethic/bootstrap/transaction_handler.ex index 00baa50833..9d460d1209 100644 --- a/lib/archethic/bootstrap/transaction_handler.ex +++ b/lib/archethic/bootstrap/transaction_handler.ex @@ -78,7 +78,9 @@ defmodule Archethic.Bootstrap.TransactionHandler do defp await_confirmation(tx_address, [node | rest]) do case P2P.send_message(node, %GetTransactionSummary{address: tx_address}) do {:ok, - %TransactionSummaryMessage{transaction_summary: %TransactionSummary{address: ^tx_address}}} -> + %TransactionSummaryMessage{ + transaction_summary: %TransactionSummary{address: ^tx_address} + }} -> :ok {:ok, %NotFound{}} -> diff --git a/lib/archethic/crypto.ex b/lib/archethic/crypto.ex index 041f886788..21df8ab286 100755 --- a/lib/archethic/crypto.ex +++ b/lib/archethic/crypto.ex @@ -1141,22 +1141,26 @@ defmodule Archethic.Crypto do """ @spec get_root_ca_public_key(key()) :: binary() def get_root_ca_public_key(<>) do - case Keyword.get(@certification_public_keys, ID.to_origin(origin_id)) do - nil -> - "no_key" - - # Only for dev - [] -> - "" - - curves when is_list(curves) -> - case Keyword.get(curves, ID.to_curve(curve)) do - nil -> - "no_key" - - public_key -> - public_key - end + if System.get_env("ARCHETHIC_NETWORK_TYPE") == "testnet" do + "" + else + case Keyword.get(@certification_public_keys, ID.to_origin(origin_id)) do + nil -> + "no_key" + + # Only for dev + [] -> + "" + + curves when is_list(curves) -> + case Keyword.get(curves, ID.to_curve(curve)) do + nil -> + "no_key" + + public_key -> + public_key + end + end end end diff --git a/lib/archethic/db/embedded_impl/chain_index.ex b/lib/archethic/db/embedded_impl/chain_index.ex index 8731cf4ee8..2b92276ca1 100644 --- a/lib/archethic/db/embedded_impl/chain_index.ex +++ b/lib/archethic/db/embedded_impl/chain_index.ex @@ -40,7 +40,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do {:ok, %{db_path: db_path}} end - def code_change("1.0.7", state, _extra) do + def code_change("1.0.6", state, _extra) do ## We start the unstarted LRU cache cache_max_size = Application.get_env(:archethic, Archethic.DB.ChainIndex.MaxCacheSize) LRU.start_link(@archetic_db_tx_index_cache, cache_max_size) diff --git a/lib/archethic/election/hypergeometric_distribution.ex b/lib/archethic/election/hypergeometric_distribution.ex index 1012404966..ecc2ddf740 100644 --- a/lib/archethic/election/hypergeometric_distribution.ex +++ b/lib/archethic/election/hypergeometric_distribution.ex @@ -19,7 +19,9 @@ defmodule Archethic.Election.HypergeometricDistribution do alias Archethic.P2P.Node alias Archethic.PubSub - @executable Application.app_dir(:archethic, "/priv/c_dist/hypergeometric_distribution") + defp executable() do + Application.app_dir(:archethic, "/priv/c_dist/hypergeometric_distribution") + end def start_link(opts \\ []) do GenServer.start_link(__MODULE__, opts, name: __MODULE__) @@ -114,7 +116,7 @@ defmodule Archethic.Election.HypergeometricDistribution do defp start_simulation_task(nb_nodes) do Task.async(fn -> - pid = Port.open({:spawn_executable, @executable}, args: [Integer.to_string(nb_nodes)]) + pid = Port.open({:spawn_executable, executable()}, args: [Integer.to_string(nb_nodes)]) receive do {^pid, {:data, data}} -> @@ -134,10 +136,10 @@ defmodule Archethic.Election.HypergeometricDistribution do iex> HypergeometricDistribution.run_simulation(5) 5 - + iex> HypergeometricDistribution.run_simulation(20) 19 - + iex> HypergeometricDistribution.run_simulation(40) 37 diff --git a/lib/archethic/governance/code/cicd/docker/cicd.ex b/lib/archethic/governance/code/cicd/docker/cicd.ex index 9b7bb33f2c..a6e33ebb8f 100644 --- a/lib/archethic/governance/code/cicd/docker/cicd.ex +++ b/lib/archethic/governance/code/cicd/docker/cicd.ex @@ -170,21 +170,9 @@ defmodule Archethic.Governance.Code.CICD.Docker do :ok end - @releases "/opt/code/_build/dev/rel/archethic_node/releases" - @release "archethic_node.tar.gz" - - defp testnet_prepare(dir, address, version) do - ci = container_name(address) - - with :ok <- File.mkdir_p!(dir), - {_, 0} <- docker(["cp", "#{ci}:#{@releases}/#{version}/#{@release}", dir]) do - :ok - else - _ -> :error - end - end - @marker Application.compile_env(:archethic, :marker) + @releases "/opt/code/_build/prod/rel/archethic_node/releases" + @release "archethic_node.tar.gz" defp do_run_docker_testnet(%Proposal{address: address, version: version}) do address_encoded = Base.encode16(address) @@ -193,24 +181,28 @@ defmodule Archethic.Governance.Code.CICD.Docker do dir = temp_dir("utn-#{address_encoded}-") nb_nodes = 5 - compose_prefix = Path.basename(dir) - validator_container = "#{compose_prefix}_validator_1" - validator_continue = ["ash", "-c", "echo 'yes' > /proc/1/fd/0"] + compose_prefix = + dir + |> Path.basename() + |> String.downcase() + + validator_container = "#{compose_prefix}-validator-1" - nodes = 1..nb_nodes |> Enum.map(&"#{compose_prefix}_node#{&1}_1") + nodes = 1..nb_nodes |> Enum.map(&"#{compose_prefix}-node#{&1}-1") with :ok <- Logger.info("#{dir} Prepare", address: address_encoded), :ok <- testnet_prepare(dir, address, version), :ok <- Logger.info("#{dir} Start", address: address_encoded), - {_, 0} <- testnet_start(dir, nb_nodes), + %{cmd: {_, 0}, testnet: _testnet} <- testnet_start(dir, nb_nodes), # wait until the validator is ready for upgrade :ok <- Logger.info("#{dir} Part I", address: address_encoded), {:ok, _} <- wait_for_marker(validator_container, @marker), :ok <- Logger.info("#{dir} Upgrade", address: address_encoded), true <- testnet_upgrade(dir, nodes, version), :ok <- Logger.info("#{dir} Part II", address: address_encoded), - {_, 0} <- docker_exec(validator_container, validator_continue), - 0 <- docker_wait(validator_container, System.monotonic_time(:second)) do + # {_, 0} <- validator_continue(validator_container, testnet), + 0 <- + docker_wait(validator_container, System.monotonic_time(:second)) do testnet_cleanup(dir, 0, address_encoded) else _ -> @@ -218,11 +210,37 @@ defmodule Archethic.Governance.Code.CICD.Docker do end end - defp testnet_cleanup(dir, code, address_encoded) do - Logger.info("#{dir} Cleanup", address: address_encoded) - System.cmd("docker-compose", ["-f", compose_file(dir), "down"], @cmd_options) - File.rm_rf!(dir) - code + defp testnet_prepare(dir, address, version) do + ci = container_name(address) + + with :ok <- File.mkdir_p!(dir), + {_, 0} <- docker(["cp", "#{ci}:#{@releases}/#{version}/#{@release}", dir]) do + :ok + else + _ -> :error + end + end + + @subnet "172.16.100.0/24" + + defp testnet_start(dir, nb_nodes) do + compose = compose_file(dir) + options = [image: "archethic-cd", dir: dir, src: @src_dir, persist: false] + + Stream.iterate(@subnet, &Subnet.next/1) + |> Stream.take(123) + |> Stream.map(fn subnet -> + testnet = Testnet.from(nb_nodes, Keyword.put(options, :subnet, subnet)) + + with :ok <- Testnet.create!(testnet, dir) do + %{ + testnet: testnet, + cmd: System.cmd("docker-compose", ["-f", compose, "up", "-d"], @cmd_options) + } + end + end) + |> Stream.filter(&(elem(&1[:cmd], 1) == 0)) + |> Enum.at(0) end defp testnet_upgrade(dir, containers, version) do @@ -255,26 +273,7 @@ defmodule Archethic.Governance.Code.CICD.Docker do result end - @subnet "172.16.100.0/24" - - defp testnet_start(dir, nb_nodes) do - compose = compose_file(dir) - options = [image: "archethic-cd", dir: dir, src: @src_dir, persist: false] - - Stream.iterate(@subnet, &Subnet.next/1) - |> Stream.take(123) - |> Stream.map(fn subnet -> - testnet = Testnet.from(nb_nodes, Keyword.put(options, :subnet, subnet)) - - with :ok <- Testnet.create!(testnet, dir) do - System.cmd("docker-compose", ["-f", compose, "up", "-d"], @cmd_options) - end - end) - |> Stream.filter(&(elem(&1, 1) == 0)) - |> Enum.at(0) - end - - defp wait_for_marker(container_name, marker, timeout \\ 600_000) do + defp wait_for_marker(container_name, marker, timeout \\ 60_000) do args = ["logs", container_name, "--follow", "--tail", "10"] opts = [:binary, :use_stdio, :stderr_to_stdout, line: 8192, args: args] @@ -311,9 +310,33 @@ defmodule Archethic.Governance.Code.CICD.Docker do end end + # defp validator_continue(validator_container, testnet) do + # uninodes = extract_ip_addresses_from_testnet_config(testnet) + + # validator_command = ["./bin/archethic_node", "validate" | uninodes] ++ ["--phase=2"] + + # docker_exec(validator_container, validator_command) + # end + + defp testnet_cleanup(dir, code, address_encoded) do + Logger.info("#{dir} Cleanup", address: address_encoded) + System.cmd("docker-compose", ["-f", compose_file(dir), "down"], @cmd_options) + File.rm_rf!(dir) + code + end + + # defp extract_ip_addresses_from_testnet_config([ + # _, + # {"docker-compose.json", %{services: services}} + # ]) do + # services + # |> Enum.filter(fn {service_name, _} -> String.contains?(service_name, "node") end) + # |> Enum.map(fn {_, %{environment: %{"ARCHETHIC_STATIC_IP" => ip}}} -> ip end) + # end + defp compose_file(dir), do: Path.join(dir, "docker-compose.json") - defp docker_exec(container_name, cmd), do: docker(["exec", container_name] ++ cmd) + # defp docker_exec(container_name, cmd), do: docker(["exec", container_name] ++ cmd) defp temp_dir(prefix, tmp \\ System.tmp_dir!()) do {_mega, sec, micro} = :os.timestamp() diff --git a/lib/archethic/governance/code/proposal/validator.ex b/lib/archethic/governance/code/proposal/validator.ex index 6d4d5a4f0c..f782b05980 100644 --- a/lib/archethic/governance/code/proposal/validator.ex +++ b/lib/archethic/governance/code/proposal/validator.ex @@ -6,28 +6,49 @@ defmodule Archethic.Governance.Code.Proposal.Validator do """ require Logger - alias Archethic.Utils alias Archethic.Utils.Regression @marker Application.compile_env(:archethic, :marker) - def run(nodes) do + def run(nodes, _) do start = System.monotonic_time(:second) with true <- Regression.nodes_up?(nodes), :ok <- Regression.run_benchmarks(nodes, phase: :before), - :ok <- await_upgrade(nodes), + :ok <- put_marker(nodes), + _ <- Process.sleep(3 * 60 * 1000), :ok <- Regression.run_benchmarks(nodes, phase: :after), :ok <- Regression.run_playbooks(nodes), {:ok, metrics} <- Regression.get_metrics("collector", 9090, 5 + System.monotonic_time(:second) - start) do - File.write!(Utils.mut_dir("metrics"), :erlang.term_to_iovec(metrics)) + write_metrics(metrics) end end - defp await_upgrade(args) do - IO.puts("#{@marker} | #{inspect(args)}") - Port.open({:fd, 0, 1}, [:out, {:line, 256}]) |> Port.command("") - IO.gets("Continue? ") |> IO.puts() + # def run(nodes, 1) do + # with true <- Regression.nodes_up?(nodes), + # :ok <- Regression.run_benchmarks(nodes, phase: :before), + # :ok <- put_marker(nodes) do + # Process.sleep(2 * 60 * 1000) + # end + # end + + # def run(nodes, 2) do + # start = System.monotonic_time(:second) + + # with :ok <- Regression.run_benchmarks(nodes, phase: :after), + # :ok <- Regression.run_playbooks(nodes), + # {:ok, metrics} <- + # Regression.get_metrics("collector", 9090, 5 + System.monotonic_time(:second) - start) do + # write_metrics(metrics) + # end + # end + + defp put_marker(args), do: IO.puts("#{@marker} | #{inspect(args)}") + + defp write_metrics(metrics) do + path = "/opt/data/metrics" + File.write!(path, :erlang.term_to_iovec(metrics)) + File.chmod!(path, 777) end end diff --git a/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex b/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex index 77e3e4aed0..07b2a7ac18 100644 --- a/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex +++ b/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex @@ -3,11 +3,13 @@ defmodule Archethic.Networking.IPLookup.NATDiscovery.MiniUPNP do require Logger - @upnpc Application.app_dir(:archethic, "priv/c_dist/upnpc") + def upnpc() do + Application.app_dir(:archethic, "priv/c_dist/upnpc") + end @spec get_node_ip() :: {:ok, :inet.ip_address()} | {:error, any()} def get_node_ip do - case System.cmd(@upnpc, ["-s"]) do + case System.cmd(upnpc(), ["-s"]) do {output, 0} -> [[_, ip]] = Regex.scan(~r/ExternalIPAddress = ([0-9.]*)/, output, capture: :all) @@ -29,7 +31,7 @@ defmodule Archethic.Networking.IPLookup.NATDiscovery.MiniUPNP do end defp get_local_ip do - case System.cmd(@upnpc, ["-s"]) do + case System.cmd(upnpc(), ["-s"]) do {output, 0} -> [[_, ip]] = Regex.scan(~r/Local LAN ip address : ([0-9.]*)/, output, capture: :all) @@ -49,7 +51,7 @@ defmodule Archethic.Networking.IPLookup.NATDiscovery.MiniUPNP do defp do_open_port(_local_ip, _port, 0), do: :error defp do_open_port(local_ip, port, retries) do - case System.cmd(@upnpc, map_query(local_ip, port)) do + case System.cmd(upnpc(), map_query(local_ip, port)) do {_, 0} -> :ok @@ -87,7 +89,7 @@ defmodule Archethic.Networking.IPLookup.NATDiscovery.MiniUPNP do defp handle_error(reason, _local_ip, port) do if Regex.scan(~r/ConflictInMappingEntry/, reason, capture: :all) != [] do Logger.warning("Port is employed to another host.") - System.cmd(@upnpc, revoke_query(port)) + System.cmd(upnpc(), revoke_query(port)) end end diff --git a/lib/archethic/networking/port_forwarding.ex b/lib/archethic/networking/port_forwarding.ex index 2a8f6a0ac5..981e6a9a5b 100644 --- a/lib/archethic/networking/port_forwarding.ex +++ b/lib/archethic/networking/port_forwarding.ex @@ -62,7 +62,7 @@ defmodule Archethic.Networking.PortForwarding do {:ok, port} :error -> - Logger.error("Cannot publish the a random port #{port}") + Logger.error("Cannot publish the random port #{port}") fallback(port, _force? = true, retries - 1) end diff --git a/lib/archethic/p2p/listener.ex b/lib/archethic/p2p/listener.ex index ff2f8b5fe8..67f1d87673 100644 --- a/lib/archethic/p2p/listener.ex +++ b/lib/archethic/p2p/listener.ex @@ -17,9 +17,9 @@ defmodule Archethic.P2P.Listener do def init(opts) do transport = Keyword.get(opts, :transport) port = Keyword.get(opts, :port) - PubSub.register_to_node_status() + # test(transport, port) {:ok, %{transport: transport, port: port}} end diff --git a/lib/archethic/utils/regression.ex b/lib/archethic/utils/regression.ex index e148d1e2b8..aac018a6ab 100644 --- a/lib/archethic/utils/regression.ex +++ b/lib/archethic/utils/regression.ex @@ -6,15 +6,13 @@ defmodule Archethic.Utils.Regression do alias Archethic.Utils - alias Archethic.Utils.Regression.Playbook.SmartContract alias Archethic.Utils.Regression.Playbook.UCO alias Archethic.Utils.WebClient alias Archethic.Utils.Regression.Benchmark.EndToEndValidation - alias Archethic.Utils.Regression.Benchmark.P2PMessage - @playbooks [UCO, SmartContract] - @benchmarks [P2PMessage, EndToEndValidation] + @playbooks [UCO] + @benchmarks [EndToEndValidation] def run_playbooks(nodes, opts \\ []) do Logger.debug("Running playbooks on #{inspect(nodes)} with #{inspect(opts)}") @@ -81,10 +79,15 @@ defmodule Archethic.Utils.Regression do |> Enum.all?(&(&1 == {:ok, :ok})) end - defp node_up?(node, start \\ System.monotonic_time(:millisecond), timeout \\ 5 * 60_000) + def node_up?(node, start \\ System.monotonic_time(:millisecond), timeout \\ 5 * 60_000) - defp node_up?(node, start, timeout) do - port = Application.get_env(:archethic, ArchethicWeb.Endpoint)[:http][:port] + def node_up?(node, start, timeout) do + port = + if System.get_env("ARCHETHIC_NETWORK_TYPE") == "testnet" do + 40_000 + else + Application.get_env(:archethic, ArchethicWeb.Endpoint)[:http][:port] + end case WebClient.with_connection(node, port, &WebClient.request(&1, "GET", "/up")) do {:ok, ["up"]} -> diff --git a/lib/archethic/utils/testnet.ex b/lib/archethic/utils/testnet.ex index 523e66ef48..83c2b81995 100644 --- a/lib/archethic/utils/testnet.ex +++ b/lib/archethic/utils/testnet.ex @@ -140,19 +140,19 @@ defmodule Archethic.Utils.Testnet do "- job_name: testnet\\n" <> " static_configs:\\n" <> " - targets:\\n" <> - " - node1:40000\\n" <> - " - node2:40000\\n" <> - " - node3:40000\\n\\n"}, + " - 1.2.3.2:40000\\n" <> + " - 1.2.3.3:40000\\n" <> + " - 1.2.3.4:40000\\n\\n"}, {"docker-compose.json", %{ version: "3.9", networks: %{:net => %{ipam: %{config: [%{subnet: "1.2.3.0/24"}], driver: :default}}}, services: %{ "node1" => %{ + build: %{context: "c"}, environment: %{ "ARCHETHIC_CRYPTO_SEED" => "node1", "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => "1.2.3.2:30002:00011D967D71B2E135C84206DDD108B5925A2CD99C8EBC5AB5D8FD2EC9400CE3C98A:tcp", "ARCHETHIC_STATIC_IP" => "1.2.3.2", - "ARCHETHIC_DB_HOST" => "scylladb1:9042", "ARCHETHIC_NETWORKING_IMPL" => "STATIC", "ARCHETHIC_NETWORKING_PORT_FORWARDING" => "false", "ARCHETHIC_NODE_ALLOWED_KEY_ORIGINS" => "software", @@ -165,17 +165,17 @@ defmodule Archethic.Utils.Testnet do "ARCHETHIC_SHARED_SECRETS_APPLICATION_INTERVAL" => "0 * * * * * *", "ARCHETHIC_SELF_REPAIR_SCHEDULER_INTRERVAL" => "5 * * * * * *", "ARCHETHIC_NODE_IP_VALIDATION" => "false", - "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE" + "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE", + "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY" => "", + "ARCHETHIC_CRYPTO_ROOT_CA_TPM_PUBKEY" => "", + "ARCHETHIC_NETWORK_TYPE" => "testnet", + "ARCHETHIC_THROTTLE_IP_AND_PATH" => 999999, + "ARCHETHIC_THROTTLE_IP_HIGH" => 999999, + "ARCHETHIC_THROTTLE_IP_LOW" => 999999 }, image: "i", - build: %{ context: "c"}, networks: %{:net => %{ipv4_address: "1.2.3.2"}}, command: [ - "/wait-for-tcp.sh", - "scylladb1:9042", - "--timeout=0", - "--strict", - "--", "./bin/archethic_node", "foreground" ], @@ -185,11 +185,11 @@ defmodule Archethic.Utils.Testnet do }, "node2" => %{ depends_on: ["node1"], + build: %{context: "c"}, environment: %{ "ARCHETHIC_CRYPTO_SEED" => "node2", "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => "1.2.3.2:30002:00011D967D71B2E135C84206DDD108B5925A2CD99C8EBC5AB5D8FD2EC9400CE3C98A:tcp", "ARCHETHIC_STATIC_IP" => "1.2.3.3", - "ARCHETHIC_DB_HOST" => "scylladb2:9042", "ARCHETHIC_NETWORKING_IMPL" => "STATIC", "ARCHETHIC_NETWORKING_PORT_FORWARDING" => "false", "ARCHETHIC_NODE_ALLOWED_KEY_ORIGINS" => "software", @@ -202,7 +202,13 @@ defmodule Archethic.Utils.Testnet do "ARCHETHIC_SHARED_SECRETS_APPLICATION_INTERVAL" => "0 * * * * * *", "ARCHETHIC_SELF_REPAIR_SCHEDULER_INTRERVAL" => "5 * * * * * *", "ARCHETHIC_NODE_IP_VALIDATION" => "false", - "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE" + "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE", + "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY" => "", + "ARCHETHIC_CRYPTO_ROOT_CA_TPM_PUBKEY" => "", + "ARCHETHIC_NETWORK_TYPE" => "testnet", + "ARCHETHIC_THROTTLE_IP_AND_PATH" => 999999, + "ARCHETHIC_THROTTLE_IP_HIGH" => 999999, + "ARCHETHIC_THROTTLE_IP_LOW" => 999999 }, image: "i", networks: %{:net => %{ipv4_address: "1.2.3.3"}}, @@ -212,26 +218,24 @@ defmodule Archethic.Utils.Testnet do ], command: [ "/wait-for-tcp.sh", - "scylladb2:9042", + "--host=1.2.3.2", + "--port=30002", "--timeout=0", - "--strict", "--", - "/wait-for-tcp.sh", - "node1:40000", - "--timeout=0", - "--strict", "--", + "--strict", + "--", "/wait-for-node.sh", - "http://node1:40000/up", + "http://1.2.3.2:40000/up", "./bin/archethic_node", "foreground" ] }, "node3" => %{ depends_on: ["node1"], + build: %{context: "c"}, environment: %{ "ARCHETHIC_CRYPTO_SEED" => "node3", "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => "1.2.3.2:30002:00011D967D71B2E135C84206DDD108B5925A2CD99C8EBC5AB5D8FD2EC9400CE3C98A:tcp", "ARCHETHIC_STATIC_IP" => "1.2.3.4", - "ARCHETHIC_DB_HOST" => "scylladb3:9042", "ARCHETHIC_NETWORKING_IMPL" => "STATIC", "ARCHETHIC_NETWORKING_PORT_FORWARDING" => "false", "ARCHETHIC_NODE_ALLOWED_KEY_ORIGINS" => "software", @@ -244,7 +248,13 @@ defmodule Archethic.Utils.Testnet do "ARCHETHIC_SHARED_SECRETS_APPLICATION_INTERVAL" => "0 * * * * * *", "ARCHETHIC_SELF_REPAIR_SCHEDULER_INTRERVAL" => "5 * * * * * *", "ARCHETHIC_NODE_IP_VALIDATION" => "false", - "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE" + "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE", + "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY" => "", + "ARCHETHIC_CRYPTO_ROOT_CA_TPM_PUBKEY" => "", + "ARCHETHIC_NETWORK_TYPE" => "testnet", + "ARCHETHIC_THROTTLE_IP_AND_PATH" => 999999, + "ARCHETHIC_THROTTLE_IP_HIGH" => 999999, + "ARCHETHIC_THROTTLE_IP_LOW" => 999999 }, image: "i", networks: %{:net => %{ipv4_address: "1.2.3.4"}}, @@ -254,15 +264,13 @@ defmodule Archethic.Utils.Testnet do ], command: [ "/wait-for-tcp.sh", - "scylladb3:9042", + "--host=1.2.3.2", + "--port=30002", "--timeout=0", - "--strict", "--", - "/wait-for-tcp.sh", - "node1:40000", - "--timeout=0", - "--strict", "--", + "--strict", + "--", "/wait-for-node.sh", - "http://node1:40000/up", + "http://1.2.3.2:40000/up", "./bin/archethic_node", "foreground" ] @@ -273,40 +281,26 @@ defmodule Archethic.Utils.Testnet do volumes: [".prometheus.yml:/etc/prometheus/prometheus.yml:ro"] }, "validator" => %{ - image: "archethic-node:latest", + image: "i", environment: %{ "ARCHETHIC_MUT_DIR" => "/opt/data" }, - command: ["./bin/archethic_node", "regression_test", "--validate", "node1", "node2", "node3"], + command: ["./bin/archethic_node", "validate", "1.2.3.2", "1.2.3.3", "1.2.3.4", "--phase=1"], volumes: [ "./validator_data/:/opt/data" ], - profiles: ["validate"], networks: %{:net => %{ipv4_address: "1.2.3.#{@validator_ip}"}}, }, "bench" => %{ - image: "archethic-node:latest", + image: "i", environment: %{ "ARCHETHIC_MUT_DIR" => "/opt/data" }, - command: ["./bin/archethic_node", "regression_test", "--bench", "node1", "node2", "node3"], + command: ["./bin/archethic_node", "regression_test", "--bench", "1.2.3.2", "1.2.3.3", "1.2.3.4"], volumes: [ "./bench_data/:/opt/data" ], - profiles: ["validate"], networks: %{ :net => %{ipv4_address: "1.2.3.#{@bench_ip}"}} - }, - "scylladb1" => %{ - image: "scylladb/scylla", - networks: %{:net => %{ipv4_address: "1.2.3.51"}} - }, - "scylladb2" => %{ - image: "scylladb/scylla", - networks: %{:net => %{ipv4_address: "1.2.3.52"}} - }, - "scylladb3" => %{ - image: "scylladb/scylla", - networks: %{:net => %{ipv4_address: "1.2.3.53"}} } } } @@ -323,18 +317,11 @@ defmodule Archethic.Utils.Testnet do ip = fn i -> Subnet.at(base, i) end services = nodes_from(nb_nodes, src, image, ip) - uninodes = Map.keys(services) - networks = %{:net => %{ipam: %{driver: :default, config: [%{subnet: subnet}]}}} + uninodes = + Enum.map(services, fn {_, %{environment: %{"ARCHETHIC_STATIC_IP" => ip}}} -> ip end) - databases = - 1..nb_nodes - |> Enum.reduce(%{}, fn i, acc -> - Map.put(acc, "scylladb#{i}", %{ - image: "scylladb/scylla", - networks: %{:net => %{ipv4_address: ip.(i + 50)}} - }) - end) + networks = %{:net => %{ipam: %{driver: :default, config: [%{subnet: subnet}]}}} services = services @@ -344,19 +331,19 @@ defmodule Archethic.Utils.Testnet do volumes: [".prometheus.yml:/etc/prometheus/prometheus.yml:ro"] }) |> Map.put("validator", %{ - image: "archethic-node:latest", + image: image, environment: %{ "ARCHETHIC_MUT_DIR" => "/opt/data" }, - command: ["./bin/archethic_node", "regression_test", "--validate" | uninodes], + command: ["./bin/archethic_node", "validate" | uninodes] ++ ["--phase=1"], volumes: [ "./validator_data/:/opt/data" ], - profiles: ["validate"], + # profiles: ["validate"], networks: %{:net => %{ipv4_address: ip.(@validator_ip)}} }) |> Map.put("bench", %{ - image: "archethic-node:latest", + image: image, environment: %{ "ARCHETHIC_MUT_DIR" => "/opt/data" }, @@ -364,10 +351,9 @@ defmodule Archethic.Utils.Testnet do volumes: [ "./bench_data/:/opt/data" ], - profiles: ["validate"], + # profiles: ["validate"], networks: %{:net => %{ipv4_address: ip.(@bench_ip)}} }) - |> Map.merge(databases) compose = %{version: "3.9", services: services, networks: networks} @@ -419,6 +405,8 @@ defmodule Archethic.Utils.Testnet do end defp to_node(1, src, image, ip) do + node_1_ip_address = ip.(1 + 1) + {"node1", %{ build: %{context: src}, @@ -426,9 +414,8 @@ defmodule Archethic.Utils.Testnet do environment: %{ "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE", "ARCHETHIC_CRYPTO_SEED" => "node1", - "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => seeder(ip.(1 + 1)), + "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => seeder(node_1_ip_address), "ARCHETHIC_STATIC_IP" => ip.(1 + 1), - "ARCHETHIC_DB_HOST" => "scylladb1:9042", "ARCHETHIC_NETWORKING_IMPL" => "STATIC", "ARCHETHIC_NETWORKING_PORT_FORWARDING" => "false", "ARCHETHIC_NODE_ALLOWED_KEY_ORIGINS" => "software", @@ -440,18 +427,19 @@ defmodule Archethic.Utils.Testnet do "ARCHETHIC_SHARED_SECRETS_RENEWAL_SCHEDULER_INTERVAL" => "40 * * * * * *", "ARCHETHIC_SHARED_SECRETS_APPLICATION_INTERVAL" => "0 * * * * * *", "ARCHETHIC_SELF_REPAIR_SCHEDULER_INTRERVAL" => "5 * * * * * *", - "ARCHETHIC_NODE_IP_VALIDATION" => "false" + "ARCHETHIC_NODE_IP_VALIDATION" => "false", + "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY" => "", + "ARCHETHIC_CRYPTO_ROOT_CA_TPM_PUBKEY" => "", + "ARCHETHIC_NETWORK_TYPE" => "testnet", + "ARCHETHIC_THROTTLE_IP_HIGH" => 999_999, + "ARCHETHIC_THROTTLE_IP_LOW" => 999_999, + "ARCHETHIC_THROTTLE_IP_AND_PATH" => 999_999 }, volumes: [ "#{Path.join([src, "/scripts/wait-for-tcp.sh"])}:/wait-for-tcp.sh:ro" ], networks: %{:net => %{ipv4_address: ip.(1 + 1)}}, command: [ - "/wait-for-tcp.sh", - "scylladb1:9042", - "--timeout=0", - "--strict", - "--", "./bin/archethic_node", "foreground" ] @@ -459,16 +447,19 @@ defmodule Archethic.Utils.Testnet do end defp to_node(n, src, image, ip) do + node_1_ip_address = ip.(1 + 1) + ip_address = ip.(n + 1) + {"node#{n}", %{ + build: %{context: src}, image: image, depends_on: ["node1"], environment: %{ "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE", "ARCHETHIC_CRYPTO_SEED" => "node#{n}", - "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => seeder(ip.(1 + 1)), - "ARCHETHIC_STATIC_IP" => ip.(n + 1), - "ARCHETHIC_DB_HOST" => "scylladb#{n}:9042", + "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => seeder(node_1_ip_address), + "ARCHETHIC_STATIC_IP" => ip_address, "ARCHETHIC_NETWORKING_IMPL" => "STATIC", "ARCHETHIC_NETWORKING_PORT_FORWARDING" => "false", "ARCHETHIC_NODE_ALLOWED_KEY_ORIGINS" => "software", @@ -480,26 +471,28 @@ defmodule Archethic.Utils.Testnet do "ARCHETHIC_SHARED_SECRETS_RENEWAL_SCHEDULER_INTERVAL" => "40 * * * * * *", "ARCHETHIC_SHARED_SECRETS_APPLICATION_INTERVAL" => "0 * * * * * *", "ARCHETHIC_SELF_REPAIR_SCHEDULER_INTRERVAL" => "5 * * * * * *", - "ARCHETHIC_NODE_IP_VALIDATION" => "false" + "ARCHETHIC_NODE_IP_VALIDATION" => "false", + "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY" => "", + "ARCHETHIC_CRYPTO_ROOT_CA_TPM_PUBKEY" => "", + "ARCHETHIC_NETWORK_TYPE" => "testnet", + "ARCHETHIC_THROTTLE_IP_HIGH" => 999_999, + "ARCHETHIC_THROTTLE_IP_LOW" => 999_999, + "ARCHETHIC_THROTTLE_IP_AND_PATH" => 999_999 }, - networks: %{:net => %{ipv4_address: ip.(n + 1)}}, + networks: %{:net => %{ipv4_address: ip_address}}, volumes: [ "#{Path.join([src, "/scripts/wait-for-tcp.sh"])}:/wait-for-tcp.sh:ro", "#{Path.join([src, "/scripts/wait-for-node.sh"])}:/wait-for-node.sh:ro" ], command: [ "/wait-for-tcp.sh", - "scylladb#{n}:9042", - "--timeout=0", - "--strict", - "--", - "/wait-for-tcp.sh", - "node1:40000", + "--host=#{node_1_ip_address}", + "--port=#{p2p_port()}", "--timeout=0", "--strict", "--", "/wait-for-node.sh", - "http://node1:#{web_port()}/up", + "http://#{node_1_ip_address}:#{web_port()}/up", "./bin/archethic_node", "foreground" ] diff --git a/lib/archethic_web/plugs/plug_throttle_by_ip_and_path.ex b/lib/archethic_web/plugs/plug_throttle_by_ip_and_path.ex index 237b38c08e..429e236257 100644 --- a/lib/archethic_web/plugs/plug_throttle_by_ip_and_path.ex +++ b/lib/archethic_web/plugs/plug_throttle_by_ip_and_path.ex @@ -10,6 +10,8 @@ defmodule ArchethicWeb.PlugThrottleByIPandPath do rule "Throttle by IP and Path", conn do [period: period, limit: limit] = Application.get_env(:archethic, :throttle)[:by_ip_and_path] + limit = String.to_integer(limit) + throttle({conn.remote_ip, conn.path_info}, period: period, limit: limit, diff --git a/lib/archethic_web/plugs/plug_throttle_by_ip_high.ex b/lib/archethic_web/plugs/plug_throttle_by_ip_high.ex index fb3083ab49..2dc9f5551c 100644 --- a/lib/archethic_web/plugs/plug_throttle_by_ip_high.ex +++ b/lib/archethic_web/plugs/plug_throttle_by_ip_high.ex @@ -9,6 +9,8 @@ defmodule ArchethicWeb.PlugThrottleByIPHigh do rule "Throttle by IP", conn do [period: period, limit: limit] = Application.get_env(:archethic, :throttle)[:by_ip_high] + limit = String.to_integer(limit) + throttle(conn.remote_ip, period: period, limit: limit, diff --git a/lib/archethic_web/plugs/plug_throttle_by_ip_low.ex b/lib/archethic_web/plugs/plug_throttle_by_ip_low.ex index 9a38210c38..ca3e9f61fe 100644 --- a/lib/archethic_web/plugs/plug_throttle_by_ip_low.ex +++ b/lib/archethic_web/plugs/plug_throttle_by_ip_low.ex @@ -9,6 +9,8 @@ defmodule ArchethicWeb.PlugThrottleByIPLow do rule "Throttle by IP", conn do [period: period, limit: limit] = Application.get_env(:archethic, :throttle)[:by_ip_low] + limit = String.to_integer(limit) + throttle(conn.remote_ip, period: period, limit: limit, diff --git a/lib/mix/tasks/validate.ex b/lib/mix/tasks/validate.ex new file mode 100644 index 0000000000..a34d60da18 --- /dev/null +++ b/lib/mix/tasks/validate.ex @@ -0,0 +1,52 @@ +defmodule Mix.Tasks.Archethic.Proposal.Validator do + @shortdoc "Run regression utilities to benchmark and validate nodes containing code proposal" + + @moduledoc """ + The Archethic Code Proposal Validator mix task wrapper + + ## Command line options + + * `--help` - show this help + * `--phase=1` - launch phase 1 + * `--phase=2` - launch phase 2 + + ## Example + + ```sh + mix archethic.validate localhost + ``` + + """ + use Mix.Task + + alias Archethic.Governance.Code.Proposal.Validator + + @impl Mix.Task + @spec run([binary]) :: any + def run(args) do + case OptionParser.parse!(args, + strict: [ + help: :boolean, + phase: :integer + ] + ) do + {_, []} -> + Mix.shell().cmd("mix help #{Mix.Task.task_name(__MODULE__)}") + + {parsed, nodes} -> + if parsed[:help] do + Mix.shell().cmd("mix help #{Mix.Task.task_name(__MODULE__)}") + else + :ok = maybe(parsed, :phase, &Validator.run/2, [nodes, parsed[:phase]]) + end + end + end + + defp maybe(opts, key, func, args) do + if opts[key] do + apply(func, args) + else + :ok + end + end +end diff --git a/mix.exs b/mix.exs index 2df49f29ed..27b2d9645c 100644 --- a/mix.exs +++ b/mix.exs @@ -10,7 +10,7 @@ defmodule Archethic.MixProject do deps_path: "deps", lockfile: "mix.lock", aliases: aliases(), - elixir: "~> 1.11", + elixir: "~> 1.14", start_permanent: Mix.env() == :prod, deps: deps(), compilers: [:elixir_make] ++ Mix.compilers(), @@ -73,7 +73,7 @@ defmodule Archethic.MixProject do {:benchee_html, "~> 1.0", only: :dev}, {:ex_doc, "~> 0.29", only: :dev, runtime: false}, {:git_hooks, "~> 0.7", runtime: false}, - {:credo, "~> 1.6", only: [:dev, :test], runtime: false}, + {:credo, "~> 1.6", runtime: false}, {:elixir_make, "~> 0.6", runtime: false}, {:dialyxir, "~> 1.2", runtime: false}, {:logger_file_backend, "~> 0.0.13", only: :dev}, @@ -81,7 +81,7 @@ defmodule Archethic.MixProject do {:dart_sass, "~> 0.5", runtime: Mix.env() == :dev}, # Security - {:sobelow, "~> 0.11", only: [:test, :dev], runtime: false}, + {:sobelow, "~> 0.11", runtime: false}, # Test {:mox, "~> 1.0", only: [:test]}, diff --git a/mix.lock b/mix.lock index da91904142..242cab1a5e 100644 --- a/mix.lock +++ b/mix.lock @@ -22,7 +22,7 @@ "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, "dialyxir": {:hex, :dialyxir, "1.2.0", "58344b3e87c2e7095304c81a9ae65cb68b613e28340690dfe1a5597fd08dec37", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "61072136427a851674cab81762be4dbeae7679f85b1272b6d25c3a839aff8463"}, "digital_token": {:hex, :digital_token, "0.4.0", "2ad6894d4a40be8b2890aad286ecd5745fa473fa5699d80361a8c94428edcd1f", [:mix], [{:cldr_utils, "~> 2.17", [hex: :cldr_utils, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "a178edf61d1fee5bb3c34e14b0f4ee21809ee87cade8738f87337e59e5e66e26"}, - "distillery": {:git, "https://github.com/archethic-foundation/distillery.git", "e66841c9108a5e5e9a8722a8f737596193b7bc0d", []}, + "distillery": {:git, "https://github.com/archethic-foundation/distillery.git", "eb279d7e01f4e7cb2f77d96dfa2b8dd7c760e661", []}, "earmark": {:hex, :earmark, "1.4.35", "e067aab15367c6e43230d6a7409c5230403a48b56f7dcefb3abdad75b498289e", [:mix], [{:earmark_parser, "~> 1.4.30", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "ab869cad78ebe64a62d45ee31addc52fb703c5d595868c9aa11ca38766ff9756"}, "earmark_parser": {:hex, :earmark_parser, "1.4.30", "0b938aa5b9bafd455056440cdaa2a79197ca5e693830b4a982beada840513c5f", [:mix], [], "hexpm", "3b5385c2d36b0473d0b206927b841343d25adb14f95f0110062506b300cd5a1b"}, "easy_ssl": {:hex, :easy_ssl, "1.3.0", "472256942d9dd37652a558a789a8d1cccc27e7f46352e32667d1ca46bb9e22e5", [:mix], [], "hexpm", "ce8fcb7661442713a94853282b56cee0b90c52b983a83aa6af24686d301808e1"}, diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000000..3893c248b3 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "archethic-node", + "lockfileVersion": 2, + "requires": true, + "packages": {} +} diff --git a/rel/commands/validate b/rel/commands/validate new file mode 100755 index 0000000000..f367504d17 --- /dev/null +++ b/rel/commands/validate @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +release_ctl eval --mfa "Mix.Tasks.Archethic.Proposal.Validator.run/1" --argv -- "$@" diff --git a/rel/config.exs b/rel/config.exs index 5d1f373353..485bacb766 100644 --- a/rel/config.exs +++ b/rel/config.exs @@ -33,11 +33,12 @@ environment Mix.env() do ] set overlays: [ - {:copy, "config/#{Mix.env()}.exs", "releases/<%= release_version %>/runtime_config.exs"} + {:copy, "config/#{Mix.env()}.exs", "releases/<%= release_version %>/runtime_config.exs"}, ] set commands: [ - regression_test: "rel/commands/regression_test" + regression_test: "rel/commands/regression_test", + validate: "rel/commands/validate" ] plugin Distillery.Releases.Plugin.CookieLoader diff --git a/scripts/governance/proposal_ci_job.sh b/scripts/governance/proposal_ci_job.sh index 99795d207f..ccc63d0318 100755 --- a/scripts/governance/proposal_ci_job.sh +++ b/scripts/governance/proposal_ci_job.sh @@ -6,6 +6,8 @@ PROPOSAL_ADDRESS=$1 PROPOSAL_DESCRIPTION=$2 PROPOSAL_FILENAME=./proposal.diff +export MIX_ENV=prod + echo "=== Test proposal ${PROPOSAL_ADDRESS}" tee ${PROPOSAL_FILENAME} @@ -24,5 +26,5 @@ git commit -m "${PROPOSAL_DESCRIPTION}" echo "=== Run CI" mix git_hooks.run pre_push -echo "=== Create upgrade" +echo "=== Create upgrade release" mix distillery.release --upgrade diff --git a/test/archethic/oracle_chain/scheduler_test.exs b/test/archethic/oracle_chain/scheduler_test.exs index 6d47b405c4..8b29a2a597 100644 --- a/test/archethic/oracle_chain/scheduler_test.exs +++ b/test/archethic/oracle_chain/scheduler_test.exs @@ -18,8 +18,6 @@ defmodule Archethic.OracleChain.SchedulerTest do alias Archethic.TransactionChain.Transaction.ValidationStamp alias Archethic.TransactionChain.TransactionData - alias ArchethicCache.HydratingCache - import ArchethicCase, only: [setup_before_send_tx: 0] import Mox