From 1002a0264c4fb988a767a8d0ea4e69fdbafb810f Mon Sep 17 00:00:00 2001 From: tenmoves Date: Fri, 3 Mar 2023 13:43:46 +0100 Subject: [PATCH 01/21] fix cd pipeline for code proposals --- .github/workflows/ci.yml | 117 +++++++------ Dockerfile | 21 +-- Makefile | 1 - assets/package-lock.json | 8 +- config/config.exs | 6 +- config/dev.exs | 6 +- config/prod.exs | 6 +- config/test.exs | 6 +- lib/archethic/crypto.ex | 36 ++-- lib/archethic/db/embedded_impl/chain_index.ex | 2 +- .../election/hypergeometric_distribution.ex | 10 +- .../governance/code/cicd/docker/cicd.ex | 100 +++++------ .../governance/code/proposal/validator.ex | 16 +- .../ip_lookup/nat_discovery/miniupnp.ex | 12 +- lib/archethic/networking/port_forwarding.ex | 2 +- lib/archethic/p2p/listener.ex | 2 +- lib/archethic/utils/regression.ex | 17 +- lib/archethic/utils/testnet.ex | 159 +++++++++--------- .../plugs/plug_throttle_by_ip_and_path.ex | 2 + .../plugs/plug_throttle_by_ip_high.ex | 2 + .../plugs/plug_throttle_by_ip_low.ex | 2 + lib/mix/tasks/validate.ex | 52 ++++++ mix.exs | 6 +- mix.lock | 2 +- package-lock.json | 6 + rel/commands/validate | 3 + rel/config.exs | 5 +- scripts/governance/proposal_ci_job.sh | 6 +- 28 files changed, 338 insertions(+), 275 deletions(-) create mode 100644 lib/mix/tasks/validate.ex create mode 100644 package-lock.json create mode 100755 rel/commands/validate diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8aefa25ae..b18f0eead 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,72 +2,71 @@ name: Elixir CI on: push: - branches: [ master, develop, testnet ] + branches: [master, develop, testnet] pull_request: - branches: [ master, develop, testnet ] + branches: [master, develop, testnet] env: MIX_ENV: test jobs: build: - name: Build and test runs-on: ubuntu-20.04 - + steps: - - name: Install OS Packages - uses: mstksg/get-package@2a4b48d55d72d43ca89ae58ec9ca1397d34a1c35 - with: - apt-get: libgmp-dev libssl-dev libtinfo-dev libsystemd-dev zlib1g-dev libsodium-dev autoconf-archive libcmocka0 libcmocka-dev procps iproute2 build-essential git pkg-config gcc libtool automake libssl-dev uthash-dev autoconf doxygen libjson-c-dev libini-config-dev libcurl4-openssl-dev libltdl-dev libtss2-dev tss2 - - uses: actions/checkout@v2 - - name: Set up Elixir - uses: erlef/setup-beam@988e02bfe678367a02564f65ca2e37726dc0268f - id: beam - with: - elixir-version: '1.14.1' # Define the elixir version [required] - otp-version: '25.1' # Define the OTP version [required] - - name: Restore dependencies cache - uses: actions/cache@v3 - id: mix-cache - with: - path: deps - key: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-mix-${{ hashFiles('**/mix.lock') }} - restore-keys: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-mix- - - name: Make Clean - run: mix clean - - name: Install dependencies - if: steps.mix-cache.outputs.cache-hit != 'true' - run: mix deps.get - - name: Check dependency updates - run: mix hex.outdated --within-requirements 1>/dev/null || echo 'Updates available!' - - name: Set Formatting - run: mix format --check-formatted - - name: Restore build cache - uses: actions/cache@v3 - with: - path: _build - key: ${{ runner.os }}-build-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('**/mix.lock') }} - restore-keys: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-build- - - name: Compile the codebase - run: mix compile --warnings-as-errors - - name: Set credo - run: mix credo - - name: Run Sobelow - run: mix sobelow - - name: Run tests - run: mix test --trace - - name: Retrieve PLT Cache - uses: actions/cache@v3 - id: plt-cache - with: - path: priv/plts - key: ${{ runner.os }}-plts-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('**/mix.lock') }} - restore-keys: ${{ runner.os }}-plts-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}- - - name: Create PLTs - if: steps.plt-cache.outputs.cache-hit != 'true' - run: | - mkdir -p priv/plts - mix dialyzer --plt - - name: Run dialyzer - run: mix dialyzer --no-check --ignore-exit-status + - name: Install OS Packages + uses: mstksg/get-package@2a4b48d55d72d43ca89ae58ec9ca1397d34a1c35 + with: + apt-get: libgmp-dev libssl-dev libtinfo-dev libsystemd-dev zlib1g-dev libsodium-dev autoconf-archive libcmocka0 libcmocka-dev procps iproute2 build-essential git pkg-config gcc libtool automake libssl-dev uthash-dev autoconf doxygen libjson-c-dev libini-config-dev libcurl4-openssl-dev libltdl-dev libtss2-dev tss2 + - uses: actions/checkout@v2 + - name: Set up Elixir + uses: erlef/setup-beam@988e02bfe678367a02564f65ca2e37726dc0268f + id: beam + with: + elixir-version: "1.14.1" # Define the elixir version [required] + otp-version: "25.1" # Define the OTP version [required] + - name: Restore dependencies cache + uses: actions/cache@v3 + id: mix-cache + with: + path: deps + key: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-mix-${{ hashFiles('**/mix.lock') }} + restore-keys: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-mix- + - name: Make Clean + run: mix clean + - name: Install dependencies + if: steps.mix-cache.outputs.cache-hit != 'true' + run: mix deps.get + - name: Check dependency updates + run: mix hex.outdated --within-requirements 1>/dev/null || echo 'Updates available!' + - name: Set Formatting + run: mix format --check-formatted + - name: Restore build cache + uses: actions/cache@v3 + with: + path: _build + key: ${{ runner.os }}-build-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('**/mix.lock') }} + restore-keys: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-build- + - name: Compile the codebase + run: mix compile --warnings-as-errors + - name: Set credo + run: mix credo + - name: Run Sobelow + run: mix sobelow + - name: Run tests + run: mix test --trace + - name: Retrieve PLT Cache + uses: actions/cache@v3 + id: plt-cache + with: + path: priv/plts + key: ${{ runner.os }}-plts-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('**/mix.lock') }} + restore-keys: ${{ runner.os }}-plts-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}- + - name: Create PLTs + if: steps.plt-cache.outputs.cache-hit != 'true' + run: | + mkdir -p priv/plts + mix dialyzer --plt + - name: Run dialyzer + run: mix dialyzer --no-check --ignore-exit-status diff --git a/Dockerfile b/Dockerfile index ef70f11f3..70cc15ddc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,21 +23,20 @@ ARG MIX_ENV=prod RUN apk add --no-cache --update \ build-base \ + grep \ bash \ gcc \ make \ g++ \ - libexecinfo-dev \ - libexecinfo \ git \ npm \ python3 \ wget \ openssl \ libsodium-dev \ + libexecinfo-dev \ gmp-dev - # Install hex and rebar RUN mix local.rebar --force \ && mix local.hex --if-missing --force @@ -60,23 +59,15 @@ RUN git config user.name aebot \ && git config user.email aebot@archethic.net \ && git remote add origin https://github.com/archethic-foundation/archethic-node -# Install Dart Sass -RUN npm install -g sass - -# build Sass -> CSS -RUN cd assets && \ - sass --no-source-map --style=compressed css/app.scss ../priv/static/css/app.css && cd - - # build release -RUN mix do assets.deploy, distillery.release - +RUN mix assets.deploy +RUN MIX_ENV=prod mix distillery.release # gen PLT RUN if [ $with_tests -eq 1 ]; then mix git_hooks.run pre_push ;fi # Install RUN mkdir -p /opt/app \ - && cd /opt/app \ - && tar zxf /opt/code/_build/${MIX_ENV}/rel/archethic_node/releases/*/archethic_node.tar.gz + && tar zxf /opt/code/_build/${MIX_ENV}/rel/archethic_node/releases/*/archethic_node.tar.gz -C /opt/app CMD /opt/app/bin/archethic_node foreground ################################################################################ @@ -85,7 +76,7 @@ FROM archethic-ci as build FROM elixir:1.14.1-alpine -RUN apk add --no-cache --update bash git openssl libsodium +RUN apk add --no-cache --update bash git openssl libsodium libexecinfo COPY --from=build /opt/app /opt/app COPY --from=build /opt/code/.git /opt/code/.git diff --git a/Makefile b/Makefile index a839a6360..1d153fec4 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,6 @@ ifeq ($(TPM_INSTALLED),0) $(CC) src/c/crypto/tpm/keygen.c src/c/crypto/tpm/lib.c -o priv/c_dist/tpm_keygen -I src/c/crypto/tpm/lib.h $(TPMFLAGS) endif - clean: rm -f priv/c_dist/* mix archethic.clean_db diff --git a/assets/package-lock.json b/assets/package-lock.json index f6e722c9d..ff4e0a796 100644 --- a/assets/package-lock.json +++ b/assets/package-lock.json @@ -24,13 +24,15 @@ } }, "../deps/phoenix": { - "version": "0.0.1" + "version": "1.6.15", + "license": "MIT" }, "../deps/phoenix_html": { - "version": "0.0.1" + "version": "3.2.0" }, "../deps/phoenix_live_view": { - "version": "0.0.1" + "version": "0.18.11", + "license": "MIT" }, "node_modules/@ampproject/remapping": { "version": "2.2.0", diff --git a/config/config.exs b/config/config.exs index 8c883d538..c3ac66f6e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -12,9 +12,9 @@ config :git_hooks, {:cmd, "mix credo"}, {:cmd, "mix sobelow"}, {:cmd, "mix knigge.verify"}, - {:cmd, "mix test --trace"}, - {:cmd, "mix dialyzer"}, - {:cmd, "mix check.updates"} + # {:cmd, "mix test --trace"}, + {:cmd, "mix dialyzer"} + # {:cmd, "mix check.updates"} ] ] ] diff --git a/config/dev.exs b/config/dev.exs index f50c59bf7..2169b4973 100755 --- a/config/dev.exs +++ b/config/dev.exs @@ -167,13 +167,13 @@ config :archethic, ArchethicWeb.Endpoint, config :archethic, :throttle, by_ip_high: [ period: 1000, - limit: 5_000 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_HIGH", "5000") ], by_ip_low: [ period: 1000, - limit: 5_000 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_LOW", "5000") ], by_ip_and_path: [ period: 1000, - limit: 5_000 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_AND_PATH", "5000") ] diff --git a/config/prod.exs b/config/prod.exs index 828205dd4..21f8f697d 100755 --- a/config/prod.exs +++ b/config/prod.exs @@ -273,13 +273,13 @@ config :archethic, ArchethicWeb.Endpoint, config :archethic, :throttle, by_ip_high: [ period: 1000, - limit: 500 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_HIGH", "500") ], by_ip_low: [ period: 1000, - limit: 20 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_LOW", "20") ], by_ip_and_path: [ period: 1000, - limit: 20 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_AND_PATH", "20") ] diff --git a/config/test.exs b/config/test.exs index 1c323e56a..d5b81db1f 100755 --- a/config/test.exs +++ b/config/test.exs @@ -179,13 +179,13 @@ config :archethic, ArchethicWeb.Endpoint, config :archethic, :throttle, by_ip_high: [ period: 1000, - limit: 5_000 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_HIGH", "5000") ], by_ip_low: [ period: 1000, - limit: 5_000 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_LOW", "5000") ], by_ip_and_path: [ period: 1000, - limit: 5_000 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_AND_PATH", "5000") ] diff --git a/lib/archethic/crypto.ex b/lib/archethic/crypto.ex index 041f88678..21df8ab28 100755 --- a/lib/archethic/crypto.ex +++ b/lib/archethic/crypto.ex @@ -1141,22 +1141,26 @@ defmodule Archethic.Crypto do """ @spec get_root_ca_public_key(key()) :: binary() def get_root_ca_public_key(<>) do - case Keyword.get(@certification_public_keys, ID.to_origin(origin_id)) do - nil -> - "no_key" - - # Only for dev - [] -> - "" - - curves when is_list(curves) -> - case Keyword.get(curves, ID.to_curve(curve)) do - nil -> - "no_key" - - public_key -> - public_key - end + if System.get_env("ARCHETHIC_NETWORK_TYPE") == "testnet" do + "" + else + case Keyword.get(@certification_public_keys, ID.to_origin(origin_id)) do + nil -> + "no_key" + + # Only for dev + [] -> + "" + + curves when is_list(curves) -> + case Keyword.get(curves, ID.to_curve(curve)) do + nil -> + "no_key" + + public_key -> + public_key + end + end end end diff --git a/lib/archethic/db/embedded_impl/chain_index.ex b/lib/archethic/db/embedded_impl/chain_index.ex index c7d4a397b..e451036a8 100644 --- a/lib/archethic/db/embedded_impl/chain_index.ex +++ b/lib/archethic/db/embedded_impl/chain_index.ex @@ -40,7 +40,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do {:ok, %{db_path: db_path}} end - def code_change("1.0.7", state, _extra) do + def code_change("1.0.6", state, _extra) do ## We start the unstarted LRU cache cache_max_size = Application.get_env(:archethic, Archethic.DB.ChainIndex.MaxCacheSize) LRU.start_link(@archetic_db_tx_index_cache, cache_max_size) diff --git a/lib/archethic/election/hypergeometric_distribution.ex b/lib/archethic/election/hypergeometric_distribution.ex index 101240496..ecc2ddf74 100644 --- a/lib/archethic/election/hypergeometric_distribution.ex +++ b/lib/archethic/election/hypergeometric_distribution.ex @@ -19,7 +19,9 @@ defmodule Archethic.Election.HypergeometricDistribution do alias Archethic.P2P.Node alias Archethic.PubSub - @executable Application.app_dir(:archethic, "/priv/c_dist/hypergeometric_distribution") + defp executable() do + Application.app_dir(:archethic, "/priv/c_dist/hypergeometric_distribution") + end def start_link(opts \\ []) do GenServer.start_link(__MODULE__, opts, name: __MODULE__) @@ -114,7 +116,7 @@ defmodule Archethic.Election.HypergeometricDistribution do defp start_simulation_task(nb_nodes) do Task.async(fn -> - pid = Port.open({:spawn_executable, @executable}, args: [Integer.to_string(nb_nodes)]) + pid = Port.open({:spawn_executable, executable()}, args: [Integer.to_string(nb_nodes)]) receive do {^pid, {:data, data}} -> @@ -134,10 +136,10 @@ defmodule Archethic.Election.HypergeometricDistribution do iex> HypergeometricDistribution.run_simulation(5) 5 - + iex> HypergeometricDistribution.run_simulation(20) 19 - + iex> HypergeometricDistribution.run_simulation(40) 37 diff --git a/lib/archethic/governance/code/cicd/docker/cicd.ex b/lib/archethic/governance/code/cicd/docker/cicd.ex index 9b7bb33f2..7a788102c 100644 --- a/lib/archethic/governance/code/cicd/docker/cicd.ex +++ b/lib/archethic/governance/code/cicd/docker/cicd.ex @@ -170,21 +170,9 @@ defmodule Archethic.Governance.Code.CICD.Docker do :ok end - @releases "/opt/code/_build/dev/rel/archethic_node/releases" - @release "archethic_node.tar.gz" - - defp testnet_prepare(dir, address, version) do - ci = container_name(address) - - with :ok <- File.mkdir_p!(dir), - {_, 0} <- docker(["cp", "#{ci}:#{@releases}/#{version}/#{@release}", dir]) do - :ok - else - _ -> :error - end - end - @marker Application.compile_env(:archethic, :marker) + @releases "/opt/code/_build/prod/rel/archethic_node/releases" + @release "archethic_node.tar.gz" defp do_run_docker_testnet(%Proposal{address: address, version: version}) do address_encoded = Base.encode16(address) @@ -193,24 +181,28 @@ defmodule Archethic.Governance.Code.CICD.Docker do dir = temp_dir("utn-#{address_encoded}-") nb_nodes = 5 - compose_prefix = Path.basename(dir) - validator_container = "#{compose_prefix}_validator_1" - validator_continue = ["ash", "-c", "echo 'yes' > /proc/1/fd/0"] + compose_prefix = + dir + |> Path.basename() + |> String.downcase() - nodes = 1..nb_nodes |> Enum.map(&"#{compose_prefix}_node#{&1}_1") + validator_container = "#{compose_prefix}-validator-1" + + nodes = 1..nb_nodes |> Enum.map(&"#{compose_prefix}-node#{&1}-1") with :ok <- Logger.info("#{dir} Prepare", address: address_encoded), :ok <- testnet_prepare(dir, address, version), :ok <- Logger.info("#{dir} Start", address: address_encoded), - {_, 0} <- testnet_start(dir, nb_nodes), + %{cmd: {_, 0}, testnet: _testnet} <- testnet_start(dir, nb_nodes), # wait until the validator is ready for upgrade :ok <- Logger.info("#{dir} Part I", address: address_encoded), {:ok, _} <- wait_for_marker(validator_container, @marker), :ok <- Logger.info("#{dir} Upgrade", address: address_encoded), true <- testnet_upgrade(dir, nodes, version), :ok <- Logger.info("#{dir} Part II", address: address_encoded), - {_, 0} <- docker_exec(validator_container, validator_continue), - 0 <- docker_wait(validator_container, System.monotonic_time(:second)) do + # {_, 0} <- validator_continue(validator_container, testnet), + 0 <- + docker_wait(validator_container, System.monotonic_time(:second)) do testnet_cleanup(dir, 0, address_encoded) else _ -> @@ -218,11 +210,37 @@ defmodule Archethic.Governance.Code.CICD.Docker do end end - defp testnet_cleanup(dir, code, address_encoded) do - Logger.info("#{dir} Cleanup", address: address_encoded) - System.cmd("docker-compose", ["-f", compose_file(dir), "down"], @cmd_options) - File.rm_rf!(dir) - code + defp testnet_prepare(dir, address, version) do + ci = container_name(address) + + with :ok <- File.mkdir_p!(dir), + {_, 0} <- docker(["cp", "#{ci}:#{@releases}/#{version}/#{@release}", dir]) do + :ok + else + _ -> :error + end + end + + @subnet "172.16.100.0/24" + + defp testnet_start(dir, nb_nodes) do + compose = compose_file(dir) + options = [image: "archethic-cd", dir: dir, src: @src_dir, persist: false] + + Stream.iterate(@subnet, &Subnet.next/1) + |> Stream.take(123) + |> Stream.map(fn subnet -> + testnet = Testnet.from(nb_nodes, Keyword.put(options, :subnet, subnet)) + + with :ok <- Testnet.create!(testnet, dir) do + %{ + testnet: testnet, + cmd: System.cmd("docker-compose", ["-f", compose, "up", "-d"], @cmd_options) + } + end + end) + |> Stream.filter(&(elem(&1[:cmd], 1) == 0)) + |> Enum.at(0) end defp testnet_upgrade(dir, containers, version) do @@ -255,26 +273,7 @@ defmodule Archethic.Governance.Code.CICD.Docker do result end - @subnet "172.16.100.0/24" - - defp testnet_start(dir, nb_nodes) do - compose = compose_file(dir) - options = [image: "archethic-cd", dir: dir, src: @src_dir, persist: false] - - Stream.iterate(@subnet, &Subnet.next/1) - |> Stream.take(123) - |> Stream.map(fn subnet -> - testnet = Testnet.from(nb_nodes, Keyword.put(options, :subnet, subnet)) - - with :ok <- Testnet.create!(testnet, dir) do - System.cmd("docker-compose", ["-f", compose, "up", "-d"], @cmd_options) - end - end) - |> Stream.filter(&(elem(&1, 1) == 0)) - |> Enum.at(0) - end - - defp wait_for_marker(container_name, marker, timeout \\ 600_000) do + defp wait_for_marker(container_name, marker, timeout \\ 60_000) do args = ["logs", container_name, "--follow", "--tail", "10"] opts = [:binary, :use_stdio, :stderr_to_stdout, line: 8192, args: args] @@ -311,9 +310,14 @@ defmodule Archethic.Governance.Code.CICD.Docker do end end - defp compose_file(dir), do: Path.join(dir, "docker-compose.json") + defp testnet_cleanup(dir, code, address_encoded) do + Logger.info("#{dir} Cleanup", address: address_encoded) + System.cmd("docker-compose", ["-f", compose_file(dir), "down", "--volumes"], @cmd_options) + File.rm_rf!(dir) + code + end - defp docker_exec(container_name, cmd), do: docker(["exec", container_name] ++ cmd) + defp compose_file(dir), do: Path.join(dir, "docker-compose.json") defp temp_dir(prefix, tmp \\ System.tmp_dir!()) do {_mega, sec, micro} = :os.timestamp() diff --git a/lib/archethic/governance/code/proposal/validator.ex b/lib/archethic/governance/code/proposal/validator.ex index 6d4d5a4f0..3cb0e57fe 100644 --- a/lib/archethic/governance/code/proposal/validator.ex +++ b/lib/archethic/governance/code/proposal/validator.ex @@ -6,28 +6,24 @@ defmodule Archethic.Governance.Code.Proposal.Validator do """ require Logger - alias Archethic.Utils alias Archethic.Utils.Regression @marker Application.compile_env(:archethic, :marker) - def run(nodes) do + def run(nodes, _) do start = System.monotonic_time(:second) with true <- Regression.nodes_up?(nodes), :ok <- Regression.run_benchmarks(nodes, phase: :before), - :ok <- await_upgrade(nodes), + :ok <- put_marker(nodes), + _ <- Process.sleep(3 * 60 * 1000), :ok <- Regression.run_benchmarks(nodes, phase: :after), :ok <- Regression.run_playbooks(nodes), - {:ok, metrics} <- + {:ok, _metrics} <- Regression.get_metrics("collector", 9090, 5 + System.monotonic_time(:second) - start) do - File.write!(Utils.mut_dir("metrics"), :erlang.term_to_iovec(metrics)) + :ok end end - defp await_upgrade(args) do - IO.puts("#{@marker} | #{inspect(args)}") - Port.open({:fd, 0, 1}, [:out, {:line, 256}]) |> Port.command("") - IO.gets("Continue? ") |> IO.puts() - end + defp put_marker(args), do: IO.puts("#{@marker} | #{inspect(args)}") end diff --git a/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex b/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex index 77e3e4aed..07b2a7ac1 100644 --- a/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex +++ b/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex @@ -3,11 +3,13 @@ defmodule Archethic.Networking.IPLookup.NATDiscovery.MiniUPNP do require Logger - @upnpc Application.app_dir(:archethic, "priv/c_dist/upnpc") + def upnpc() do + Application.app_dir(:archethic, "priv/c_dist/upnpc") + end @spec get_node_ip() :: {:ok, :inet.ip_address()} | {:error, any()} def get_node_ip do - case System.cmd(@upnpc, ["-s"]) do + case System.cmd(upnpc(), ["-s"]) do {output, 0} -> [[_, ip]] = Regex.scan(~r/ExternalIPAddress = ([0-9.]*)/, output, capture: :all) @@ -29,7 +31,7 @@ defmodule Archethic.Networking.IPLookup.NATDiscovery.MiniUPNP do end defp get_local_ip do - case System.cmd(@upnpc, ["-s"]) do + case System.cmd(upnpc(), ["-s"]) do {output, 0} -> [[_, ip]] = Regex.scan(~r/Local LAN ip address : ([0-9.]*)/, output, capture: :all) @@ -49,7 +51,7 @@ defmodule Archethic.Networking.IPLookup.NATDiscovery.MiniUPNP do defp do_open_port(_local_ip, _port, 0), do: :error defp do_open_port(local_ip, port, retries) do - case System.cmd(@upnpc, map_query(local_ip, port)) do + case System.cmd(upnpc(), map_query(local_ip, port)) do {_, 0} -> :ok @@ -87,7 +89,7 @@ defmodule Archethic.Networking.IPLookup.NATDiscovery.MiniUPNP do defp handle_error(reason, _local_ip, port) do if Regex.scan(~r/ConflictInMappingEntry/, reason, capture: :all) != [] do Logger.warning("Port is employed to another host.") - System.cmd(@upnpc, revoke_query(port)) + System.cmd(upnpc(), revoke_query(port)) end end diff --git a/lib/archethic/networking/port_forwarding.ex b/lib/archethic/networking/port_forwarding.ex index 0460dbb42..097431f96 100644 --- a/lib/archethic/networking/port_forwarding.ex +++ b/lib/archethic/networking/port_forwarding.ex @@ -62,7 +62,7 @@ defmodule Archethic.Networking.PortForwarding do {:ok, port} :error -> - Logger.warning("Cannot publish the a random port #{port}") + Logger.warning("Cannot publish the random port #{port}") fallback(port, _force? = true, retries - 1) end diff --git a/lib/archethic/p2p/listener.ex b/lib/archethic/p2p/listener.ex index ff2f8b5fe..67f1d8767 100644 --- a/lib/archethic/p2p/listener.ex +++ b/lib/archethic/p2p/listener.ex @@ -17,9 +17,9 @@ defmodule Archethic.P2P.Listener do def init(opts) do transport = Keyword.get(opts, :transport) port = Keyword.get(opts, :port) - PubSub.register_to_node_status() + # test(transport, port) {:ok, %{transport: transport, port: port}} end diff --git a/lib/archethic/utils/regression.ex b/lib/archethic/utils/regression.ex index e148d1e2b..aac018a6a 100644 --- a/lib/archethic/utils/regression.ex +++ b/lib/archethic/utils/regression.ex @@ -6,15 +6,13 @@ defmodule Archethic.Utils.Regression do alias Archethic.Utils - alias Archethic.Utils.Regression.Playbook.SmartContract alias Archethic.Utils.Regression.Playbook.UCO alias Archethic.Utils.WebClient alias Archethic.Utils.Regression.Benchmark.EndToEndValidation - alias Archethic.Utils.Regression.Benchmark.P2PMessage - @playbooks [UCO, SmartContract] - @benchmarks [P2PMessage, EndToEndValidation] + @playbooks [UCO] + @benchmarks [EndToEndValidation] def run_playbooks(nodes, opts \\ []) do Logger.debug("Running playbooks on #{inspect(nodes)} with #{inspect(opts)}") @@ -81,10 +79,15 @@ defmodule Archethic.Utils.Regression do |> Enum.all?(&(&1 == {:ok, :ok})) end - defp node_up?(node, start \\ System.monotonic_time(:millisecond), timeout \\ 5 * 60_000) + def node_up?(node, start \\ System.monotonic_time(:millisecond), timeout \\ 5 * 60_000) - defp node_up?(node, start, timeout) do - port = Application.get_env(:archethic, ArchethicWeb.Endpoint)[:http][:port] + def node_up?(node, start, timeout) do + port = + if System.get_env("ARCHETHIC_NETWORK_TYPE") == "testnet" do + 40_000 + else + Application.get_env(:archethic, ArchethicWeb.Endpoint)[:http][:port] + end case WebClient.with_connection(node, port, &WebClient.request(&1, "GET", "/up")) do {:ok, ["up"]} -> diff --git a/lib/archethic/utils/testnet.ex b/lib/archethic/utils/testnet.ex index 523e66ef4..83c2b8199 100644 --- a/lib/archethic/utils/testnet.ex +++ b/lib/archethic/utils/testnet.ex @@ -140,19 +140,19 @@ defmodule Archethic.Utils.Testnet do "- job_name: testnet\\n" <> " static_configs:\\n" <> " - targets:\\n" <> - " - node1:40000\\n" <> - " - node2:40000\\n" <> - " - node3:40000\\n\\n"}, + " - 1.2.3.2:40000\\n" <> + " - 1.2.3.3:40000\\n" <> + " - 1.2.3.4:40000\\n\\n"}, {"docker-compose.json", %{ version: "3.9", networks: %{:net => %{ipam: %{config: [%{subnet: "1.2.3.0/24"}], driver: :default}}}, services: %{ "node1" => %{ + build: %{context: "c"}, environment: %{ "ARCHETHIC_CRYPTO_SEED" => "node1", "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => "1.2.3.2:30002:00011D967D71B2E135C84206DDD108B5925A2CD99C8EBC5AB5D8FD2EC9400CE3C98A:tcp", "ARCHETHIC_STATIC_IP" => "1.2.3.2", - "ARCHETHIC_DB_HOST" => "scylladb1:9042", "ARCHETHIC_NETWORKING_IMPL" => "STATIC", "ARCHETHIC_NETWORKING_PORT_FORWARDING" => "false", "ARCHETHIC_NODE_ALLOWED_KEY_ORIGINS" => "software", @@ -165,17 +165,17 @@ defmodule Archethic.Utils.Testnet do "ARCHETHIC_SHARED_SECRETS_APPLICATION_INTERVAL" => "0 * * * * * *", "ARCHETHIC_SELF_REPAIR_SCHEDULER_INTRERVAL" => "5 * * * * * *", "ARCHETHIC_NODE_IP_VALIDATION" => "false", - "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE" + "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE", + "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY" => "", + "ARCHETHIC_CRYPTO_ROOT_CA_TPM_PUBKEY" => "", + "ARCHETHIC_NETWORK_TYPE" => "testnet", + "ARCHETHIC_THROTTLE_IP_AND_PATH" => 999999, + "ARCHETHIC_THROTTLE_IP_HIGH" => 999999, + "ARCHETHIC_THROTTLE_IP_LOW" => 999999 }, image: "i", - build: %{ context: "c"}, networks: %{:net => %{ipv4_address: "1.2.3.2"}}, command: [ - "/wait-for-tcp.sh", - "scylladb1:9042", - "--timeout=0", - "--strict", - "--", "./bin/archethic_node", "foreground" ], @@ -185,11 +185,11 @@ defmodule Archethic.Utils.Testnet do }, "node2" => %{ depends_on: ["node1"], + build: %{context: "c"}, environment: %{ "ARCHETHIC_CRYPTO_SEED" => "node2", "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => "1.2.3.2:30002:00011D967D71B2E135C84206DDD108B5925A2CD99C8EBC5AB5D8FD2EC9400CE3C98A:tcp", "ARCHETHIC_STATIC_IP" => "1.2.3.3", - "ARCHETHIC_DB_HOST" => "scylladb2:9042", "ARCHETHIC_NETWORKING_IMPL" => "STATIC", "ARCHETHIC_NETWORKING_PORT_FORWARDING" => "false", "ARCHETHIC_NODE_ALLOWED_KEY_ORIGINS" => "software", @@ -202,7 +202,13 @@ defmodule Archethic.Utils.Testnet do "ARCHETHIC_SHARED_SECRETS_APPLICATION_INTERVAL" => "0 * * * * * *", "ARCHETHIC_SELF_REPAIR_SCHEDULER_INTRERVAL" => "5 * * * * * *", "ARCHETHIC_NODE_IP_VALIDATION" => "false", - "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE" + "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE", + "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY" => "", + "ARCHETHIC_CRYPTO_ROOT_CA_TPM_PUBKEY" => "", + "ARCHETHIC_NETWORK_TYPE" => "testnet", + "ARCHETHIC_THROTTLE_IP_AND_PATH" => 999999, + "ARCHETHIC_THROTTLE_IP_HIGH" => 999999, + "ARCHETHIC_THROTTLE_IP_LOW" => 999999 }, image: "i", networks: %{:net => %{ipv4_address: "1.2.3.3"}}, @@ -212,26 +218,24 @@ defmodule Archethic.Utils.Testnet do ], command: [ "/wait-for-tcp.sh", - "scylladb2:9042", + "--host=1.2.3.2", + "--port=30002", "--timeout=0", - "--strict", "--", - "/wait-for-tcp.sh", - "node1:40000", - "--timeout=0", - "--strict", "--", + "--strict", + "--", "/wait-for-node.sh", - "http://node1:40000/up", + "http://1.2.3.2:40000/up", "./bin/archethic_node", "foreground" ] }, "node3" => %{ depends_on: ["node1"], + build: %{context: "c"}, environment: %{ "ARCHETHIC_CRYPTO_SEED" => "node3", "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => "1.2.3.2:30002:00011D967D71B2E135C84206DDD108B5925A2CD99C8EBC5AB5D8FD2EC9400CE3C98A:tcp", "ARCHETHIC_STATIC_IP" => "1.2.3.4", - "ARCHETHIC_DB_HOST" => "scylladb3:9042", "ARCHETHIC_NETWORKING_IMPL" => "STATIC", "ARCHETHIC_NETWORKING_PORT_FORWARDING" => "false", "ARCHETHIC_NODE_ALLOWED_KEY_ORIGINS" => "software", @@ -244,7 +248,13 @@ defmodule Archethic.Utils.Testnet do "ARCHETHIC_SHARED_SECRETS_APPLICATION_INTERVAL" => "0 * * * * * *", "ARCHETHIC_SELF_REPAIR_SCHEDULER_INTRERVAL" => "5 * * * * * *", "ARCHETHIC_NODE_IP_VALIDATION" => "false", - "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE" + "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE", + "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY" => "", + "ARCHETHIC_CRYPTO_ROOT_CA_TPM_PUBKEY" => "", + "ARCHETHIC_NETWORK_TYPE" => "testnet", + "ARCHETHIC_THROTTLE_IP_AND_PATH" => 999999, + "ARCHETHIC_THROTTLE_IP_HIGH" => 999999, + "ARCHETHIC_THROTTLE_IP_LOW" => 999999 }, image: "i", networks: %{:net => %{ipv4_address: "1.2.3.4"}}, @@ -254,15 +264,13 @@ defmodule Archethic.Utils.Testnet do ], command: [ "/wait-for-tcp.sh", - "scylladb3:9042", + "--host=1.2.3.2", + "--port=30002", "--timeout=0", - "--strict", "--", - "/wait-for-tcp.sh", - "node1:40000", - "--timeout=0", - "--strict", "--", + "--strict", + "--", "/wait-for-node.sh", - "http://node1:40000/up", + "http://1.2.3.2:40000/up", "./bin/archethic_node", "foreground" ] @@ -273,40 +281,26 @@ defmodule Archethic.Utils.Testnet do volumes: [".prometheus.yml:/etc/prometheus/prometheus.yml:ro"] }, "validator" => %{ - image: "archethic-node:latest", + image: "i", environment: %{ "ARCHETHIC_MUT_DIR" => "/opt/data" }, - command: ["./bin/archethic_node", "regression_test", "--validate", "node1", "node2", "node3"], + command: ["./bin/archethic_node", "validate", "1.2.3.2", "1.2.3.3", "1.2.3.4", "--phase=1"], volumes: [ "./validator_data/:/opt/data" ], - profiles: ["validate"], networks: %{:net => %{ipv4_address: "1.2.3.#{@validator_ip}"}}, }, "bench" => %{ - image: "archethic-node:latest", + image: "i", environment: %{ "ARCHETHIC_MUT_DIR" => "/opt/data" }, - command: ["./bin/archethic_node", "regression_test", "--bench", "node1", "node2", "node3"], + command: ["./bin/archethic_node", "regression_test", "--bench", "1.2.3.2", "1.2.3.3", "1.2.3.4"], volumes: [ "./bench_data/:/opt/data" ], - profiles: ["validate"], networks: %{ :net => %{ipv4_address: "1.2.3.#{@bench_ip}"}} - }, - "scylladb1" => %{ - image: "scylladb/scylla", - networks: %{:net => %{ipv4_address: "1.2.3.51"}} - }, - "scylladb2" => %{ - image: "scylladb/scylla", - networks: %{:net => %{ipv4_address: "1.2.3.52"}} - }, - "scylladb3" => %{ - image: "scylladb/scylla", - networks: %{:net => %{ipv4_address: "1.2.3.53"}} } } } @@ -323,18 +317,11 @@ defmodule Archethic.Utils.Testnet do ip = fn i -> Subnet.at(base, i) end services = nodes_from(nb_nodes, src, image, ip) - uninodes = Map.keys(services) - networks = %{:net => %{ipam: %{driver: :default, config: [%{subnet: subnet}]}}} + uninodes = + Enum.map(services, fn {_, %{environment: %{"ARCHETHIC_STATIC_IP" => ip}}} -> ip end) - databases = - 1..nb_nodes - |> Enum.reduce(%{}, fn i, acc -> - Map.put(acc, "scylladb#{i}", %{ - image: "scylladb/scylla", - networks: %{:net => %{ipv4_address: ip.(i + 50)}} - }) - end) + networks = %{:net => %{ipam: %{driver: :default, config: [%{subnet: subnet}]}}} services = services @@ -344,19 +331,19 @@ defmodule Archethic.Utils.Testnet do volumes: [".prometheus.yml:/etc/prometheus/prometheus.yml:ro"] }) |> Map.put("validator", %{ - image: "archethic-node:latest", + image: image, environment: %{ "ARCHETHIC_MUT_DIR" => "/opt/data" }, - command: ["./bin/archethic_node", "regression_test", "--validate" | uninodes], + command: ["./bin/archethic_node", "validate" | uninodes] ++ ["--phase=1"], volumes: [ "./validator_data/:/opt/data" ], - profiles: ["validate"], + # profiles: ["validate"], networks: %{:net => %{ipv4_address: ip.(@validator_ip)}} }) |> Map.put("bench", %{ - image: "archethic-node:latest", + image: image, environment: %{ "ARCHETHIC_MUT_DIR" => "/opt/data" }, @@ -364,10 +351,9 @@ defmodule Archethic.Utils.Testnet do volumes: [ "./bench_data/:/opt/data" ], - profiles: ["validate"], + # profiles: ["validate"], networks: %{:net => %{ipv4_address: ip.(@bench_ip)}} }) - |> Map.merge(databases) compose = %{version: "3.9", services: services, networks: networks} @@ -419,6 +405,8 @@ defmodule Archethic.Utils.Testnet do end defp to_node(1, src, image, ip) do + node_1_ip_address = ip.(1 + 1) + {"node1", %{ build: %{context: src}, @@ -426,9 +414,8 @@ defmodule Archethic.Utils.Testnet do environment: %{ "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE", "ARCHETHIC_CRYPTO_SEED" => "node1", - "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => seeder(ip.(1 + 1)), + "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => seeder(node_1_ip_address), "ARCHETHIC_STATIC_IP" => ip.(1 + 1), - "ARCHETHIC_DB_HOST" => "scylladb1:9042", "ARCHETHIC_NETWORKING_IMPL" => "STATIC", "ARCHETHIC_NETWORKING_PORT_FORWARDING" => "false", "ARCHETHIC_NODE_ALLOWED_KEY_ORIGINS" => "software", @@ -440,18 +427,19 @@ defmodule Archethic.Utils.Testnet do "ARCHETHIC_SHARED_SECRETS_RENEWAL_SCHEDULER_INTERVAL" => "40 * * * * * *", "ARCHETHIC_SHARED_SECRETS_APPLICATION_INTERVAL" => "0 * * * * * *", "ARCHETHIC_SELF_REPAIR_SCHEDULER_INTRERVAL" => "5 * * * * * *", - "ARCHETHIC_NODE_IP_VALIDATION" => "false" + "ARCHETHIC_NODE_IP_VALIDATION" => "false", + "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY" => "", + "ARCHETHIC_CRYPTO_ROOT_CA_TPM_PUBKEY" => "", + "ARCHETHIC_NETWORK_TYPE" => "testnet", + "ARCHETHIC_THROTTLE_IP_HIGH" => 999_999, + "ARCHETHIC_THROTTLE_IP_LOW" => 999_999, + "ARCHETHIC_THROTTLE_IP_AND_PATH" => 999_999 }, volumes: [ "#{Path.join([src, "/scripts/wait-for-tcp.sh"])}:/wait-for-tcp.sh:ro" ], networks: %{:net => %{ipv4_address: ip.(1 + 1)}}, command: [ - "/wait-for-tcp.sh", - "scylladb1:9042", - "--timeout=0", - "--strict", - "--", "./bin/archethic_node", "foreground" ] @@ -459,16 +447,19 @@ defmodule Archethic.Utils.Testnet do end defp to_node(n, src, image, ip) do + node_1_ip_address = ip.(1 + 1) + ip_address = ip.(n + 1) + {"node#{n}", %{ + build: %{context: src}, image: image, depends_on: ["node1"], environment: %{ "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE", "ARCHETHIC_CRYPTO_SEED" => "node#{n}", - "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => seeder(ip.(1 + 1)), - "ARCHETHIC_STATIC_IP" => ip.(n + 1), - "ARCHETHIC_DB_HOST" => "scylladb#{n}:9042", + "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => seeder(node_1_ip_address), + "ARCHETHIC_STATIC_IP" => ip_address, "ARCHETHIC_NETWORKING_IMPL" => "STATIC", "ARCHETHIC_NETWORKING_PORT_FORWARDING" => "false", "ARCHETHIC_NODE_ALLOWED_KEY_ORIGINS" => "software", @@ -480,26 +471,28 @@ defmodule Archethic.Utils.Testnet do "ARCHETHIC_SHARED_SECRETS_RENEWAL_SCHEDULER_INTERVAL" => "40 * * * * * *", "ARCHETHIC_SHARED_SECRETS_APPLICATION_INTERVAL" => "0 * * * * * *", "ARCHETHIC_SELF_REPAIR_SCHEDULER_INTRERVAL" => "5 * * * * * *", - "ARCHETHIC_NODE_IP_VALIDATION" => "false" + "ARCHETHIC_NODE_IP_VALIDATION" => "false", + "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY" => "", + "ARCHETHIC_CRYPTO_ROOT_CA_TPM_PUBKEY" => "", + "ARCHETHIC_NETWORK_TYPE" => "testnet", + "ARCHETHIC_THROTTLE_IP_HIGH" => 999_999, + "ARCHETHIC_THROTTLE_IP_LOW" => 999_999, + "ARCHETHIC_THROTTLE_IP_AND_PATH" => 999_999 }, - networks: %{:net => %{ipv4_address: ip.(n + 1)}}, + networks: %{:net => %{ipv4_address: ip_address}}, volumes: [ "#{Path.join([src, "/scripts/wait-for-tcp.sh"])}:/wait-for-tcp.sh:ro", "#{Path.join([src, "/scripts/wait-for-node.sh"])}:/wait-for-node.sh:ro" ], command: [ "/wait-for-tcp.sh", - "scylladb#{n}:9042", - "--timeout=0", - "--strict", - "--", - "/wait-for-tcp.sh", - "node1:40000", + "--host=#{node_1_ip_address}", + "--port=#{p2p_port()}", "--timeout=0", "--strict", "--", "/wait-for-node.sh", - "http://node1:#{web_port()}/up", + "http://#{node_1_ip_address}:#{web_port()}/up", "./bin/archethic_node", "foreground" ] diff --git a/lib/archethic_web/plugs/plug_throttle_by_ip_and_path.ex b/lib/archethic_web/plugs/plug_throttle_by_ip_and_path.ex index 237b38c08..429e23625 100644 --- a/lib/archethic_web/plugs/plug_throttle_by_ip_and_path.ex +++ b/lib/archethic_web/plugs/plug_throttle_by_ip_and_path.ex @@ -10,6 +10,8 @@ defmodule ArchethicWeb.PlugThrottleByIPandPath do rule "Throttle by IP and Path", conn do [period: period, limit: limit] = Application.get_env(:archethic, :throttle)[:by_ip_and_path] + limit = String.to_integer(limit) + throttle({conn.remote_ip, conn.path_info}, period: period, limit: limit, diff --git a/lib/archethic_web/plugs/plug_throttle_by_ip_high.ex b/lib/archethic_web/plugs/plug_throttle_by_ip_high.ex index fb3083ab4..2dc9f5551 100644 --- a/lib/archethic_web/plugs/plug_throttle_by_ip_high.ex +++ b/lib/archethic_web/plugs/plug_throttle_by_ip_high.ex @@ -9,6 +9,8 @@ defmodule ArchethicWeb.PlugThrottleByIPHigh do rule "Throttle by IP", conn do [period: period, limit: limit] = Application.get_env(:archethic, :throttle)[:by_ip_high] + limit = String.to_integer(limit) + throttle(conn.remote_ip, period: period, limit: limit, diff --git a/lib/archethic_web/plugs/plug_throttle_by_ip_low.ex b/lib/archethic_web/plugs/plug_throttle_by_ip_low.ex index 9a38210c3..ca3e9f61f 100644 --- a/lib/archethic_web/plugs/plug_throttle_by_ip_low.ex +++ b/lib/archethic_web/plugs/plug_throttle_by_ip_low.ex @@ -9,6 +9,8 @@ defmodule ArchethicWeb.PlugThrottleByIPLow do rule "Throttle by IP", conn do [period: period, limit: limit] = Application.get_env(:archethic, :throttle)[:by_ip_low] + limit = String.to_integer(limit) + throttle(conn.remote_ip, period: period, limit: limit, diff --git a/lib/mix/tasks/validate.ex b/lib/mix/tasks/validate.ex new file mode 100644 index 000000000..a34d60da1 --- /dev/null +++ b/lib/mix/tasks/validate.ex @@ -0,0 +1,52 @@ +defmodule Mix.Tasks.Archethic.Proposal.Validator do + @shortdoc "Run regression utilities to benchmark and validate nodes containing code proposal" + + @moduledoc """ + The Archethic Code Proposal Validator mix task wrapper + + ## Command line options + + * `--help` - show this help + * `--phase=1` - launch phase 1 + * `--phase=2` - launch phase 2 + + ## Example + + ```sh + mix archethic.validate localhost + ``` + + """ + use Mix.Task + + alias Archethic.Governance.Code.Proposal.Validator + + @impl Mix.Task + @spec run([binary]) :: any + def run(args) do + case OptionParser.parse!(args, + strict: [ + help: :boolean, + phase: :integer + ] + ) do + {_, []} -> + Mix.shell().cmd("mix help #{Mix.Task.task_name(__MODULE__)}") + + {parsed, nodes} -> + if parsed[:help] do + Mix.shell().cmd("mix help #{Mix.Task.task_name(__MODULE__)}") + else + :ok = maybe(parsed, :phase, &Validator.run/2, [nodes, parsed[:phase]]) + end + end + end + + defp maybe(opts, key, func, args) do + if opts[key] do + apply(func, args) + else + :ok + end + end +end diff --git a/mix.exs b/mix.exs index aae295bdf..8960794b2 100644 --- a/mix.exs +++ b/mix.exs @@ -10,7 +10,7 @@ defmodule Archethic.MixProject do deps_path: "deps", lockfile: "mix.lock", aliases: aliases(), - elixir: "~> 1.11", + elixir: "~> 1.14", start_permanent: Mix.env() == :prod, deps: deps(), compilers: [:elixir_make] ++ Mix.compilers(), @@ -74,7 +74,7 @@ defmodule Archethic.MixProject do {:benchee_html, "~> 1.0", only: :dev}, {:ex_doc, "~> 0.29", only: :dev, runtime: false}, {:git_hooks, "~> 0.7", runtime: false}, - {:credo, "~> 1.6", only: [:dev, :test], runtime: false}, + {:credo, "~> 1.6", runtime: false}, {:elixir_make, "~> 0.6", runtime: false}, {:dialyxir, "~> 1.2", runtime: false}, {:logger_file_backend, "~> 0.0.13", only: :dev}, @@ -82,7 +82,7 @@ defmodule Archethic.MixProject do {:dart_sass, "~> 0.5", runtime: Mix.env() == :dev}, # Security - {:sobelow, "~> 0.11", only: [:test, :dev], runtime: false}, + {:sobelow, "~> 0.11", runtime: false}, # Test {:mox, "~> 1.0", only: [:test]}, diff --git a/mix.lock b/mix.lock index afc743fe4..2d07d4540 100644 --- a/mix.lock +++ b/mix.lock @@ -22,7 +22,7 @@ "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, "dialyxir": {:hex, :dialyxir, "1.2.0", "58344b3e87c2e7095304c81a9ae65cb68b613e28340690dfe1a5597fd08dec37", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "61072136427a851674cab81762be4dbeae7679f85b1272b6d25c3a839aff8463"}, "digital_token": {:hex, :digital_token, "0.4.0", "2ad6894d4a40be8b2890aad286ecd5745fa473fa5699d80361a8c94428edcd1f", [:mix], [{:cldr_utils, "~> 2.17", [hex: :cldr_utils, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "a178edf61d1fee5bb3c34e14b0f4ee21809ee87cade8738f87337e59e5e66e26"}, - "distillery": {:git, "https://github.com/archethic-foundation/distillery.git", "a85f23713c2914096293b25f256e578d40514189", []}, + "distillery": {:git, "https://github.com/archethic-foundation/distillery.git", "eb279d7e01f4e7cb2f77d96dfa2b8dd7c760e661", []}, "earmark": {:hex, :earmark, "1.4.35", "e067aab15367c6e43230d6a7409c5230403a48b56f7dcefb3abdad75b498289e", [:mix], [{:earmark_parser, "~> 1.4.30", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "ab869cad78ebe64a62d45ee31addc52fb703c5d595868c9aa11ca38766ff9756"}, "earmark_parser": {:hex, :earmark_parser, "1.4.30", "0b938aa5b9bafd455056440cdaa2a79197ca5e693830b4a982beada840513c5f", [:mix], [], "hexpm", "3b5385c2d36b0473d0b206927b841343d25adb14f95f0110062506b300cd5a1b"}, "easy_ssl": {:hex, :easy_ssl, "1.3.0", "472256942d9dd37652a558a789a8d1cccc27e7f46352e32667d1ca46bb9e22e5", [:mix], [], "hexpm", "ce8fcb7661442713a94853282b56cee0b90c52b983a83aa6af24686d301808e1"}, diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 000000000..3893c248b --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "archethic-node", + "lockfileVersion": 2, + "requires": true, + "packages": {} +} diff --git a/rel/commands/validate b/rel/commands/validate new file mode 100755 index 000000000..f367504d1 --- /dev/null +++ b/rel/commands/validate @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +release_ctl eval --mfa "Mix.Tasks.Archethic.Proposal.Validator.run/1" --argv -- "$@" diff --git a/rel/config.exs b/rel/config.exs index 8aef01704..3725bb148 100644 --- a/rel/config.exs +++ b/rel/config.exs @@ -32,11 +32,12 @@ environment Mix.env() do ] set overlays: [ - {:copy, "config/#{Mix.env()}.exs", "releases/<%= release_version %>/runtime_config.exs"} + {:copy, "config/#{Mix.env()}.exs", "releases/<%= release_version %>/runtime_config.exs"}, ] set commands: [ - regression_test: "rel/commands/regression_test" + regression_test: "rel/commands/regression_test", + validate: "rel/commands/validate" ] plugin Distillery.Releases.Plugin.CookieLoader diff --git a/scripts/governance/proposal_ci_job.sh b/scripts/governance/proposal_ci_job.sh index 99795d207..e07c7c54e 100755 --- a/scripts/governance/proposal_ci_job.sh +++ b/scripts/governance/proposal_ci_job.sh @@ -18,11 +18,11 @@ git apply ${PROPOSAL_FILENAME} echo "=== git add files" git add --all -echo "=== git commit " +echo "=== git commit" git commit -m "${PROPOSAL_DESCRIPTION}" echo "=== Run CI" mix git_hooks.run pre_push -echo "=== Create upgrade" -mix distillery.release --upgrade +echo "=== Create upgrade release" +MIX_ENV=prod mix distillery.release --upgrade From 91cdce7814e319f4646da80ac95375a95daeca2d Mon Sep 17 00:00:00 2001 From: tenmoves Date: Wed, 5 Apr 2023 16:12:20 +0000 Subject: [PATCH 02/21] review feedbacks --- Dockerfile | 4 +-- config/config.exs | 6 ++-- config/prod.exs | 23 +++++++----- config/test.exs | 21 +++++++---- lib/archethic/crypto.ex | 36 +++++++++---------- .../governance/code/cicd/docker/cicd.ex | 2 ++ .../ip_lookup/nat_discovery/miniupnp.ex | 2 +- lib/mix/tasks/regression.ex | 21 +++++------ lib/mix/tasks/utils.ex | 11 ++++++ lib/mix/tasks/validate.ex | 15 ++++---- 10 files changed, 81 insertions(+), 60 deletions(-) create mode 100644 lib/mix/tasks/utils.ex diff --git a/Dockerfile b/Dockerfile index 70cc15ddc..0e7a2c342 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM elixir:1.14.1-alpine AS archethic-ci -ARG skip_tests=0 +ARG with_tests=1 ARG MIX_ENV=prod # CI @@ -19,7 +19,7 @@ ARG MIX_ENV=prod # - code # - release -# running TESTNET with release upgrade should ??? +ENV ARCHETHIC_NETWORK_TYPE=testnet RUN apk add --no-cache --update \ build-base \ diff --git a/config/config.exs b/config/config.exs index c3ac66f6e..8c883d538 100644 --- a/config/config.exs +++ b/config/config.exs @@ -12,9 +12,9 @@ config :git_hooks, {:cmd, "mix credo"}, {:cmd, "mix sobelow"}, {:cmd, "mix knigge.verify"}, - # {:cmd, "mix test --trace"}, - {:cmd, "mix dialyzer"} - # {:cmd, "mix check.updates"} + {:cmd, "mix test --trace"}, + {:cmd, "mix dialyzer"}, + {:cmd, "mix check.updates"} ] ] ] diff --git a/config/prod.exs b/config/prod.exs index 21f8f697d..7d0940864 100755 --- a/config/prod.exs +++ b/config/prod.exs @@ -89,14 +89,21 @@ config :archethic, Archethic.BeaconChain.SummaryTimer, config :archethic, Archethic.Crypto, root_ca_public_keys: [ - software: [ - secp256r1: - System.get_env( - "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY", - "04F0FE701A03CE375A6E57ADBE0255808812036571C1424DB2779C77E8B4A9BA80A15B118E8E7465EE2E94094E59C4B3F7177E99063AF1B19BFCC4D7E1AC3F89DD" - ) - |> Base.decode16!(case: :mixed) - ], + software: + case System.get_env("ARCHETHIC_NETWORK_TYPE") do + "testnet" -> + [] + + _ -> + [ + secp256r1: + System.get_env( + "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY", + "04F0FE701A03CE375A6E57ADBE0255808812036571C1424DB2779C77E8B4A9BA80A15B118E8E7465EE2E94094E59C4B3F7177E99063AF1B19BFCC4D7E1AC3F89DD" + ) + |> Base.decode16!(case: :mixed) + ] + end, tpm: [ secp256r1: System.get_env( diff --git a/config/test.exs b/config/test.exs index d5b81db1f..f704eece7 100755 --- a/config/test.exs +++ b/config/test.exs @@ -49,13 +49,20 @@ config :archethic, Archethic.Crypto, 186, 81, 243, 58, 13, 198, 129, 169, 33, 179, 201, 50, 49, 67, 38, 156, 38, 199, 97, 59, 70, 95, 28, 35, 233, 21, 230>> ], - software: [ - secp256r1: - <<4, 210, 136, 107, 189, 140, 118, 86, 124, 217, 244, 69, 111, 61, 56, 224, 56, 150, 230, - 194, 203, 81, 213, 212, 220, 19, 1, 180, 114, 44, 230, 149, 21, 125, 69, 206, 32, 173, - 186, 81, 243, 58, 13, 198, 129, 169, 33, 179, 201, 50, 49, 67, 38, 156, 38, 199, 97, 59, - 70, 95, 28, 35, 233, 21, 230>> - ] + software: + case System.get_env("ARCHETHIC_NETWORK_TYPE") do + "testnet" -> + [] + + _ -> + [ + secp256r1: + <<4, 210, 136, 107, 189, 140, 118, 86, 124, 217, 244, 69, 111, 61, 56, 224, 56, 150, + 230, 194, 203, 81, 213, 212, 220, 19, 1, 180, 114, 44, 230, 149, 21, 125, 69, 206, + 32, 173, 186, 81, 243, 58, 13, 198, 129, 169, 33, 179, 201, 50, 49, 67, 38, 156, + 38, 199, 97, 59, 70, 95, 28, 35, 233, 21, 230>> + ] + end ] config :archethic, MockCrypto.NodeKeystore, enabled: false diff --git a/lib/archethic/crypto.ex b/lib/archethic/crypto.ex index 21df8ab28..e47f6cec4 100755 --- a/lib/archethic/crypto.ex +++ b/lib/archethic/crypto.ex @@ -1141,26 +1141,22 @@ defmodule Archethic.Crypto do """ @spec get_root_ca_public_key(key()) :: binary() def get_root_ca_public_key(<>) do - if System.get_env("ARCHETHIC_NETWORK_TYPE") == "testnet" do - "" - else - case Keyword.get(@certification_public_keys, ID.to_origin(origin_id)) do - nil -> - "no_key" - - # Only for dev - [] -> - "" - - curves when is_list(curves) -> - case Keyword.get(curves, ID.to_curve(curve)) do - nil -> - "no_key" - - public_key -> - public_key - end - end + case Keyword.get(@certification_public_keys, ID.to_origin(origin_id)) do + nil -> + "no_key" + + # Only for dev and testnet + [] -> + "" + + curves when is_list(curves) -> + case Keyword.get(curves, ID.to_curve(curve)) do + nil -> + "no_key" + + public_key -> + public_key + end end end diff --git a/lib/archethic/governance/code/cicd/docker/cicd.ex b/lib/archethic/governance/code/cicd/docker/cicd.ex index 7a788102c..198ca98c6 100644 --- a/lib/archethic/governance/code/cicd/docker/cicd.ex +++ b/lib/archethic/governance/code/cicd/docker/cicd.ex @@ -310,6 +310,8 @@ defmodule Archethic.Governance.Code.CICD.Docker do end end + defp testnet_cleanup(_dir, 1, _address_encoded), do: 1 + defp testnet_cleanup(dir, code, address_encoded) do Logger.info("#{dir} Cleanup", address: address_encoded) System.cmd("docker-compose", ["-f", compose_file(dir), "down", "--volumes"], @cmd_options) diff --git a/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex b/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex index 07b2a7ac1..6e803c761 100644 --- a/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex +++ b/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex @@ -3,7 +3,7 @@ defmodule Archethic.Networking.IPLookup.NATDiscovery.MiniUPNP do require Logger - def upnpc() do + defp upnpc() do Application.app_dir(:archethic, "priv/c_dist/upnpc") end diff --git a/lib/mix/tasks/regression.ex b/lib/mix/tasks/regression.ex index 67867fa26..69b1002bf 100644 --- a/lib/mix/tasks/regression.ex +++ b/lib/mix/tasks/regression.ex @@ -23,6 +23,7 @@ defmodule Mix.Tasks.Archethic.Regression do use Mix.Task alias Archethic.Utils.Regression + alias Mix.Tasks.Utils @impl Mix.Task def run(args) do @@ -43,17 +44,17 @@ defmodule Mix.Tasks.Archethic.Regression do Mix.shell().cmd("mix help #{Mix.Task.task_name(__MODULE__)}") else true = Regression.nodes_up?(nodes) - :ok = maybe(parsed, :bench, &Regression.run_benchmarks/1, [nodes]) - :ok = maybe(parsed, :playbook, &Regression.run_playbooks/1, [nodes]) - end - end - end - defp maybe(opts, key, func, args) do - if opts[key] do - apply(func, args) - else - :ok + :ok = + Utils.apply_function_if_key_exists(parsed, :bench, &Regression.run_benchmarks/1, [ + nodes + ]) + + :ok = + Utils.apply_function_if_key_exists(parsed, :playbook, &Regression.run_playbooks/1, [ + nodes + ]) + end end end end diff --git a/lib/mix/tasks/utils.ex b/lib/mix/tasks/utils.ex new file mode 100644 index 000000000..5601f1f88 --- /dev/null +++ b/lib/mix/tasks/utils.ex @@ -0,0 +1,11 @@ +defmodule Mix.Tasks.Utils do + @moduledoc false + + def apply_function_if_key_exists(opts, key, func, args) do + if opts[key] do + apply(func, args) + else + :ok + end + end +end diff --git a/lib/mix/tasks/validate.ex b/lib/mix/tasks/validate.ex index a34d60da1..d917b2956 100644 --- a/lib/mix/tasks/validate.ex +++ b/lib/mix/tasks/validate.ex @@ -20,6 +20,7 @@ defmodule Mix.Tasks.Archethic.Proposal.Validator do use Mix.Task alias Archethic.Governance.Code.Proposal.Validator + alias Mix.Tasks.Utils @impl Mix.Task @spec run([binary]) :: any @@ -37,16 +38,12 @@ defmodule Mix.Tasks.Archethic.Proposal.Validator do if parsed[:help] do Mix.shell().cmd("mix help #{Mix.Task.task_name(__MODULE__)}") else - :ok = maybe(parsed, :phase, &Validator.run/2, [nodes, parsed[:phase]]) + :ok = + Utils.apply_function_if_key_exists(parsed, :phase, &Validator.run/2, [ + nodes, + parsed[:phase] + ]) end end end - - defp maybe(opts, key, func, args) do - if opts[key] do - apply(func, args) - else - :ok - end - end end From 8fc6be8cc660011d59b882630de278ce5886ee60 Mon Sep 17 00:00:00 2001 From: tenmoves Date: Tue, 11 Apr 2023 07:59:37 +0000 Subject: [PATCH 03/21] change code_change in chain index from 1.0.6 to 1.0.7 --- lib/archethic/db/embedded_impl/chain_index.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/archethic/db/embedded_impl/chain_index.ex b/lib/archethic/db/embedded_impl/chain_index.ex index e451036a8..c7d4a397b 100644 --- a/lib/archethic/db/embedded_impl/chain_index.ex +++ b/lib/archethic/db/embedded_impl/chain_index.ex @@ -40,7 +40,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do {:ok, %{db_path: db_path}} end - def code_change("1.0.6", state, _extra) do + def code_change("1.0.7", state, _extra) do ## We start the unstarted LRU cache cache_max_size = Application.get_env(:archethic, Archethic.DB.ChainIndex.MaxCacheSize) LRU.start_link(@archetic_db_tx_index_cache, cache_max_size) From 63e9dbef9e7cad57b4d396b1d6e9e77f16d268f6 Mon Sep 17 00:00:00 2001 From: tenmoves Date: Fri, 21 Apr 2023 13:58:05 +0000 Subject: [PATCH 04/21] fixing ci pipeline --- Dockerfile | 2 -- scripts/governance/proposal_ci_job.sh | 32 +++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0e7a2c342..84ce58e73 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,8 +62,6 @@ RUN git config user.name aebot \ # build release RUN mix assets.deploy RUN MIX_ENV=prod mix distillery.release -# gen PLT -RUN if [ $with_tests -eq 1 ]; then mix git_hooks.run pre_push ;fi # Install RUN mkdir -p /opt/app \ diff --git a/scripts/governance/proposal_ci_job.sh b/scripts/governance/proposal_ci_job.sh index e07c7c54e..e9b8bc630 100755 --- a/scripts/governance/proposal_ci_job.sh +++ b/scripts/governance/proposal_ci_job.sh @@ -21,8 +21,36 @@ git add --all echo "=== git commit" git commit -m "${PROPOSAL_DESCRIPTION}" -echo "=== Run CI" -mix git_hooks.run pre_push +echo "=== Run CI START ===" + +echo "=== Run CI -- Part 1 -- clean " +mix clean + +echo "=== Run CI -- Part 2 -- format" +mix format --check-formatted + +echo "=== Run CI -- 3 -- compile" +mix compile --warnings-as-errors + +echo "=== Run CI -- 4 -- credo" +mix credo + +echo "=== Run CI -- 5 -- sobelow" +mix sobelow + +echo "=== Run CI -- 6 -- knigge" +mix knigge.verify + +echo "=== Run CI -- 7 -- test" +#MIX_ENV=test mix test --trace + +echo "=== Run CI -- 8 -- dialyzer" +#mix dialyzer + +echo "=== Run CI -- 9 -- updates" +#mix check.updates + +echo "=== Run CI DONE ===" echo "=== Create upgrade release" MIX_ENV=prod mix distillery.release --upgrade From 3af98596640b36f8d943e86e1524e145def73516 Mon Sep 17 00:00:00 2001 From: tenmoves Date: Tue, 25 Apr 2023 11:33:25 +0100 Subject: [PATCH 05/21] fixed crypto root ca certificate value in test --- config/test.exs | 21 +++++++-------------- scripts/governance/proposal_ci_job.sh | 6 +++--- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/config/test.exs b/config/test.exs index f704eece7..d5b81db1f 100755 --- a/config/test.exs +++ b/config/test.exs @@ -49,20 +49,13 @@ config :archethic, Archethic.Crypto, 186, 81, 243, 58, 13, 198, 129, 169, 33, 179, 201, 50, 49, 67, 38, 156, 38, 199, 97, 59, 70, 95, 28, 35, 233, 21, 230>> ], - software: - case System.get_env("ARCHETHIC_NETWORK_TYPE") do - "testnet" -> - [] - - _ -> - [ - secp256r1: - <<4, 210, 136, 107, 189, 140, 118, 86, 124, 217, 244, 69, 111, 61, 56, 224, 56, 150, - 230, 194, 203, 81, 213, 212, 220, 19, 1, 180, 114, 44, 230, 149, 21, 125, 69, 206, - 32, 173, 186, 81, 243, 58, 13, 198, 129, 169, 33, 179, 201, 50, 49, 67, 38, 156, - 38, 199, 97, 59, 70, 95, 28, 35, 233, 21, 230>> - ] - end + software: [ + secp256r1: + <<4, 210, 136, 107, 189, 140, 118, 86, 124, 217, 244, 69, 111, 61, 56, 224, 56, 150, 230, + 194, 203, 81, 213, 212, 220, 19, 1, 180, 114, 44, 230, 149, 21, 125, 69, 206, 32, 173, + 186, 81, 243, 58, 13, 198, 129, 169, 33, 179, 201, 50, 49, 67, 38, 156, 38, 199, 97, 59, + 70, 95, 28, 35, 233, 21, 230>> + ] ] config :archethic, MockCrypto.NodeKeystore, enabled: false diff --git a/scripts/governance/proposal_ci_job.sh b/scripts/governance/proposal_ci_job.sh index e9b8bc630..626f29c8b 100755 --- a/scripts/governance/proposal_ci_job.sh +++ b/scripts/governance/proposal_ci_job.sh @@ -42,13 +42,13 @@ echo "=== Run CI -- 6 -- knigge" mix knigge.verify echo "=== Run CI -- 7 -- test" -#MIX_ENV=test mix test --trace +MIX_ENV=test mix test --trace echo "=== Run CI -- 8 -- dialyzer" -#mix dialyzer +mix dialyzer echo "=== Run CI -- 9 -- updates" -#mix check.updates +mix check.updates echo "=== Run CI DONE ===" From 094b1da747061acdabb02a5ec09deed1414e3b62 Mon Sep 17 00:00:00 2001 From: tenmoves Date: Wed, 26 Apr 2023 11:12:59 +0100 Subject: [PATCH 06/21] fix logs redirection output --- lib/archethic/governance/code/cicd/docker/cicd.ex | 4 +++- scripts/governance/proposal_ci_job.sh | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/archethic/governance/code/cicd/docker/cicd.ex b/lib/archethic/governance/code/cicd/docker/cicd.ex index 198ca98c6..611136471 100644 --- a/lib/archethic/governance/code/cicd/docker/cicd.ex +++ b/lib/archethic/governance/code/cicd/docker/cicd.ex @@ -210,11 +210,13 @@ defmodule Archethic.Governance.Code.CICD.Docker do end end + @logfile_name "ci_logfile.txt" defp testnet_prepare(dir, address, version) do ci = container_name(address) with :ok <- File.mkdir_p!(dir), - {_, 0} <- docker(["cp", "#{ci}:#{@releases}/#{version}/#{@release}", dir]) do + {_, 0} <- docker(["cp", "#{ci}:#{@releases}/#{version}/#{@release}", dir]), + {_, 0} <- docker(["cp", "#{ci}:/opt/code/#{@logfile_name}", dir <> "/#{@logfile_name}"]) do :ok else _ -> :error diff --git a/scripts/governance/proposal_ci_job.sh b/scripts/governance/proposal_ci_job.sh index 626f29c8b..6f1504b9b 100755 --- a/scripts/governance/proposal_ci_job.sh +++ b/scripts/governance/proposal_ci_job.sh @@ -1,5 +1,8 @@ #!/bin/bash +# redirect stdout/stderr to a file +exec >ci_logfile.txt 2>&1 + set -e PROPOSAL_ADDRESS=$1 From cdada59271bfb8e6161a998ae09e317115a7f429 Mon Sep 17 00:00:00 2001 From: tenmoves Date: Tue, 2 May 2023 15:08:00 +0100 Subject: [PATCH 07/21] reduce docker images size --- Dockerfile | 4 +++- .../governance/code/cicd/docker/cicd.ex | 24 ++++++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 84ce58e73..04f13d722 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,6 @@ RUN apk add --no-cache --update \ g++ \ git \ npm \ - python3 \ wget \ openssl \ libsodium-dev \ @@ -82,5 +81,8 @@ COPY --from=build /opt/code/.git /opt/code/.git WORKDIR /opt/code RUN git reset --hard +RUN rm -rf /opt/code/.git +RUN rm -rf /opt/code/priv + WORKDIR /opt/app CMD /opt/app/bin/archethic_node foreground diff --git a/lib/archethic/governance/code/cicd/docker/cicd.ex b/lib/archethic/governance/code/cicd/docker/cicd.ex index 611136471..ecdd52648 100644 --- a/lib/archethic/governance/code/cicd/docker/cicd.ex +++ b/lib/archethic/governance/code/cicd/docker/cicd.ex @@ -312,11 +312,29 @@ defmodule Archethic.Governance.Code.CICD.Docker do end end - defp testnet_cleanup(_dir, 1, _address_encoded), do: 1 - defp testnet_cleanup(dir, code, address_encoded) do Logger.info("#{dir} Cleanup", address: address_encoded) - System.cmd("docker-compose", ["-f", compose_file(dir), "down", "--volumes"], @cmd_options) + + System.cmd( + "docker-compose", + [ + "-f", + compose_file(dir), + "down", + "--volumes" + ], + @cmd_options + ) + + docker([ + "image", + "rm", + "-f", + "archethic-cd", + "archethic-ci", + "prom/prometheus" + ]) + File.rm_rf!(dir) code end From 5ddd5f94aebafe0ab28f0a6cde5a9bd998c8c6ec Mon Sep 17 00:00:00 2001 From: tenmoves Date: Fri, 3 Mar 2023 13:43:46 +0100 Subject: [PATCH 08/21] fix cd pipeline for code proposals --- Dockerfile | 1 - config/config.exs | 6 ++-- lib/archethic/crypto.ex | 36 ++++++++++--------- lib/archethic/db/embedded_impl/chain_index.ex | 2 +- .../governance/code/cicd/docker/cicd.ex | 5 +-- .../ip_lookup/nat_discovery/miniupnp.ex | 2 +- lib/mix/tasks/validate.ex | 14 +++++--- 7 files changed, 35 insertions(+), 31 deletions(-) diff --git a/Dockerfile b/Dockerfile index 04f13d722..465e8c616 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,7 +61,6 @@ RUN git config user.name aebot \ # build release RUN mix assets.deploy RUN MIX_ENV=prod mix distillery.release - # Install RUN mkdir -p /opt/app \ && tar zxf /opt/code/_build/${MIX_ENV}/rel/archethic_node/releases/*/archethic_node.tar.gz -C /opt/app diff --git a/config/config.exs b/config/config.exs index 8c883d538..c3ac66f6e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -12,9 +12,9 @@ config :git_hooks, {:cmd, "mix credo"}, {:cmd, "mix sobelow"}, {:cmd, "mix knigge.verify"}, - {:cmd, "mix test --trace"}, - {:cmd, "mix dialyzer"}, - {:cmd, "mix check.updates"} + # {:cmd, "mix test --trace"}, + {:cmd, "mix dialyzer"} + # {:cmd, "mix check.updates"} ] ] ] diff --git a/lib/archethic/crypto.ex b/lib/archethic/crypto.ex index e47f6cec4..21df8ab28 100755 --- a/lib/archethic/crypto.ex +++ b/lib/archethic/crypto.ex @@ -1141,22 +1141,26 @@ defmodule Archethic.Crypto do """ @spec get_root_ca_public_key(key()) :: binary() def get_root_ca_public_key(<>) do - case Keyword.get(@certification_public_keys, ID.to_origin(origin_id)) do - nil -> - "no_key" - - # Only for dev and testnet - [] -> - "" - - curves when is_list(curves) -> - case Keyword.get(curves, ID.to_curve(curve)) do - nil -> - "no_key" - - public_key -> - public_key - end + if System.get_env("ARCHETHIC_NETWORK_TYPE") == "testnet" do + "" + else + case Keyword.get(@certification_public_keys, ID.to_origin(origin_id)) do + nil -> + "no_key" + + # Only for dev + [] -> + "" + + curves when is_list(curves) -> + case Keyword.get(curves, ID.to_curve(curve)) do + nil -> + "no_key" + + public_key -> + public_key + end + end end end diff --git a/lib/archethic/db/embedded_impl/chain_index.ex b/lib/archethic/db/embedded_impl/chain_index.ex index c7d4a397b..e451036a8 100644 --- a/lib/archethic/db/embedded_impl/chain_index.ex +++ b/lib/archethic/db/embedded_impl/chain_index.ex @@ -40,7 +40,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do {:ok, %{db_path: db_path}} end - def code_change("1.0.7", state, _extra) do + def code_change("1.0.6", state, _extra) do ## We start the unstarted LRU cache cache_max_size = Application.get_env(:archethic, Archethic.DB.ChainIndex.MaxCacheSize) LRU.start_link(@archetic_db_tx_index_cache, cache_max_size) diff --git a/lib/archethic/governance/code/cicd/docker/cicd.ex b/lib/archethic/governance/code/cicd/docker/cicd.ex index ecdd52648..2c75989d1 100644 --- a/lib/archethic/governance/code/cicd/docker/cicd.ex +++ b/lib/archethic/governance/code/cicd/docker/cicd.ex @@ -210,13 +210,11 @@ defmodule Archethic.Governance.Code.CICD.Docker do end end - @logfile_name "ci_logfile.txt" defp testnet_prepare(dir, address, version) do ci = container_name(address) with :ok <- File.mkdir_p!(dir), - {_, 0} <- docker(["cp", "#{ci}:#{@releases}/#{version}/#{@release}", dir]), - {_, 0} <- docker(["cp", "#{ci}:/opt/code/#{@logfile_name}", dir <> "/#{@logfile_name}"]) do + {_, 0} <- docker(["cp", "#{ci}:#{@releases}/#{version}/#{@release}", dir]) do :ok else _ -> :error @@ -224,7 +222,6 @@ defmodule Archethic.Governance.Code.CICD.Docker do end @subnet "172.16.100.0/24" - defp testnet_start(dir, nb_nodes) do compose = compose_file(dir) options = [image: "archethic-cd", dir: dir, src: @src_dir, persist: false] diff --git a/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex b/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex index 6e803c761..07b2a7ac1 100644 --- a/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex +++ b/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex @@ -3,7 +3,7 @@ defmodule Archethic.Networking.IPLookup.NATDiscovery.MiniUPNP do require Logger - defp upnpc() do + def upnpc() do Application.app_dir(:archethic, "priv/c_dist/upnpc") end diff --git a/lib/mix/tasks/validate.ex b/lib/mix/tasks/validate.ex index d917b2956..ed719ef93 100644 --- a/lib/mix/tasks/validate.ex +++ b/lib/mix/tasks/validate.ex @@ -38,12 +38,16 @@ defmodule Mix.Tasks.Archethic.Proposal.Validator do if parsed[:help] do Mix.shell().cmd("mix help #{Mix.Task.task_name(__MODULE__)}") else - :ok = - Utils.apply_function_if_key_exists(parsed, :phase, &Validator.run/2, [ - nodes, - parsed[:phase] - ]) + :ok = maybe(parsed, :phase, &Validator.run/2, [nodes, parsed[:phase]]) end end end + + defp maybe(opts, key, func, args) do + if opts[key] do + apply(func, args) + else + :ok + end + end end From ce9737a7bd2fe15fad26a96de3e652be5b757df3 Mon Sep 17 00:00:00 2001 From: tenmoves Date: Wed, 5 Apr 2023 16:12:20 +0000 Subject: [PATCH 09/21] eview feedbacks --- .iex.exs | 24 +++++++++++++ Dockerfile | 5 +++ config/config.exs | 6 ++-- .../governance/code/cicd/docker/cicd.ex | 34 +++++++++++++++++-- .../governance/code/proposal/validator.ex | 8 +++-- lib/mix/tasks/validate.ex | 1 - 6 files changed, 70 insertions(+), 8 deletions(-) diff --git a/.iex.exs b/.iex.exs index 6dbc4a11a..9cb865592 100644 --- a/.iex.exs +++ b/.iex.exs @@ -13,3 +13,27 @@ alias Archethic.TransactionChain alias Archethic.TransactionChain.Transaction alias Archethic.TransactionChain.TransactionData alias Archethic.BeaconChain + + +alias Archethic.Governance.Code.CICD.Docker +prop = %Archethic.Governance.Code.Proposal{ + address: <<0, 0, 135, 189, 175, 8, 43, 160, 49, 21, 23, 241, 61, 189, 221, + 177, 45, 120, 59, 243, 80, 75, 193, 250, 119, 188, 219, 73, 209, 197, 118, + 170, 122, 142>>, + previous_public_key: <<0, 1, 62, 205, 23, 142, 29, 217, 94, 202, 136, 109, + 165, 135, 153, 205, 202, 132, 122, 116, 97, 18, 90, 34, 141, 225, 172, 138, + 16, 138, 85, 15, 41, 149>>, + timestamp: nil, + description: "\Testing code proposal\"", + changes: "diff --git a/mix.exs b/mix.exs\nindex a82c0b3c..b94d4323 100644\n--- a/mix.exs\n+++ b/mix.exs\n@@ -4,7 +4,7 @@ defmodule Archethic.MixProject do\n def project do\n [\n app: :archethic,\n- version: \"1.0.7\",\n+ version: \"1.0.8\",\n build_path: \"_build\",\n config_path: \"config/config.exs\",\n deps_path: \"deps\",\ndiff --git a/rel/appups/archethic/1.0.7_to_1.0.8.appup b/rel/appups/archethic/1.0.7_to_1.0.8.appup\nnew file mode 100644\nindex 00000000..18b6d541\n--- /dev/null\n+++ b/rel/appups/archethic/1.0.7_to_1.0.8.appup\n@@ -0,0 +1,4 @@\n+{\"1.0.8\",\n+ [{\"1.0.7\",\n+ [{load_module,'Elixir.Archethic', []}]}],\n+ [{\"1.0.7\",\n+ [{load_module,'Elixir.Archethic', []}]}]\n+}.\n", + version: "1.0.8", + files: ["mix.exs", "rel/appups/archethic/1.0.7_to_1.0.8.appup"], + approvals: [ + <<0, 0, 135, 189, 175, 8, 43, 160, 49, 21, 23, 241, 61, 189, 221, 177, 45, + 120, 59, 243, 80, 75, 193, 250, 119, 188, 219, 73, 209, 197, 118, 170, + 122, 142>>, + <<0, 0, 168, 220, 209, 92, 167, 255, 187, 168, 176, 63, 128, 210, 199, 35, + 63, 0, 252, 36, 147, 196, 130, 249, 233, 17, 132, 0, 6, 88, 232, 198, 98, + 44>> + ] +} diff --git a/Dockerfile b/Dockerfile index 465e8c616..e07eff4bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,8 @@ FROM elixir:1.14.1-alpine AS archethic-ci ARG with_tests=1 ARG MIX_ENV=prod +ARG USER_ID +ARG GROUP_ID # CI # - compile @@ -72,6 +74,9 @@ FROM archethic-ci as build FROM elixir:1.14.1-alpine +ARG USER_ID +ARG GROUP_ID + RUN apk add --no-cache --update bash git openssl libsodium libexecinfo COPY --from=build /opt/app /opt/app diff --git a/config/config.exs b/config/config.exs index c3ac66f6e..8c883d538 100644 --- a/config/config.exs +++ b/config/config.exs @@ -12,9 +12,9 @@ config :git_hooks, {:cmd, "mix credo"}, {:cmd, "mix sobelow"}, {:cmd, "mix knigge.verify"}, - # {:cmd, "mix test --trace"}, - {:cmd, "mix dialyzer"} - # {:cmd, "mix check.updates"} + {:cmd, "mix test --trace"}, + {:cmd, "mix dialyzer"}, + {:cmd, "mix check.updates"} ] ] ] diff --git a/lib/archethic/governance/code/cicd/docker/cicd.ex b/lib/archethic/governance/code/cicd/docker/cicd.ex index 2c75989d1..482ef029b 100644 --- a/lib/archethic/governance/code/cicd/docker/cicd.ex +++ b/lib/archethic/governance/code/cicd/docker/cicd.ex @@ -123,7 +123,23 @@ defmodule Archethic.Governance.Code.CICD.Docker do # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ [CI] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ defp build_ci_image do - {_, 0} = docker(["build", "-t", "archethic-ci", "--target", "archethic-ci", "."]) + {user_id, _} = System.cmd("id", ["-u"]) + {group_id, _} = System.cmd("id", ["-g"]) + + {_, 0} = + docker([ + "build", + "-t", + "archethic-ci", + "--target", + "archethic-ci", + "--build-arg", + "USER_ID=#{String.trim(user_id)}", + "--build-arg", + "GROUP_ID=#{String.trim(group_id)}", + "." + ]) + :ok end @@ -166,7 +182,21 @@ defmodule Archethic.Governance.Code.CICD.Docker do # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ [CD] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ defp build_cd_image do - {_, 0} = docker(["build", "-t", "archethic-cd", "."]) + {user_id, _} = System.cmd("id", ["-u"]) + {group_id, _} = System.cmd("id", ["-g"]) + + {_, 0} = + docker([ + "build", + "-t", + "archethic-cd", + "--build-arg", + "USER_ID=#{String.trim(user_id)}", + "--build-arg", + "GROUP_ID=#{String.trim(group_id)}", + "." + ]) + :ok end diff --git a/lib/archethic/governance/code/proposal/validator.ex b/lib/archethic/governance/code/proposal/validator.ex index 3cb0e57fe..cbad3bb50 100644 --- a/lib/archethic/governance/code/proposal/validator.ex +++ b/lib/archethic/governance/code/proposal/validator.ex @@ -6,6 +6,7 @@ defmodule Archethic.Governance.Code.Proposal.Validator do """ require Logger + alias Archethic.Utils alias Archethic.Utils.Regression @marker Application.compile_env(:archethic, :marker) @@ -19,11 +20,14 @@ defmodule Archethic.Governance.Code.Proposal.Validator do _ <- Process.sleep(3 * 60 * 1000), :ok <- Regression.run_benchmarks(nodes, phase: :after), :ok <- Regression.run_playbooks(nodes), - {:ok, _metrics} <- + {:ok, metrics} <- Regression.get_metrics("collector", 9090, 5 + System.monotonic_time(:second) - start) do - :ok + write_metrics(metrics) end end + defp write_metrics(metrics), + do: File.write!(Utils.mut_dir("metrics"), :erlang.term_to_iovec(metrics)) + defp put_marker(args), do: IO.puts("#{@marker} | #{inspect(args)}") end diff --git a/lib/mix/tasks/validate.ex b/lib/mix/tasks/validate.ex index ed719ef93..a34d60da1 100644 --- a/lib/mix/tasks/validate.ex +++ b/lib/mix/tasks/validate.ex @@ -20,7 +20,6 @@ defmodule Mix.Tasks.Archethic.Proposal.Validator do use Mix.Task alias Archethic.Governance.Code.Proposal.Validator - alias Mix.Tasks.Utils @impl Mix.Task @spec run([binary]) :: any From c88c406d7a48946d78220f52e9057ba36206f435 Mon Sep 17 00:00:00 2001 From: tenmoves Date: Fri, 3 Mar 2023 13:43:46 +0100 Subject: [PATCH 10/21] fix cd pipeline for code proposals --- config/config.exs | 6 +++--- lib/archethic/governance/code/cicd/docker/cicd.ex | 1 + lib/archethic/governance/code/proposal/validator.ex | 3 +-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/config.exs b/config/config.exs index 8c883d538..c3ac66f6e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -12,9 +12,9 @@ config :git_hooks, {:cmd, "mix credo"}, {:cmd, "mix sobelow"}, {:cmd, "mix knigge.verify"}, - {:cmd, "mix test --trace"}, - {:cmd, "mix dialyzer"}, - {:cmd, "mix check.updates"} + # {:cmd, "mix test --trace"}, + {:cmd, "mix dialyzer"} + # {:cmd, "mix check.updates"} ] ] ] diff --git a/lib/archethic/governance/code/cicd/docker/cicd.ex b/lib/archethic/governance/code/cicd/docker/cicd.ex index 482ef029b..1dbb274e3 100644 --- a/lib/archethic/governance/code/cicd/docker/cicd.ex +++ b/lib/archethic/governance/code/cicd/docker/cicd.ex @@ -252,6 +252,7 @@ defmodule Archethic.Governance.Code.CICD.Docker do end @subnet "172.16.100.0/24" + defp testnet_start(dir, nb_nodes) do compose = compose_file(dir) options = [image: "archethic-cd", dir: dir, src: @src_dir, persist: false] diff --git a/lib/archethic/governance/code/proposal/validator.ex b/lib/archethic/governance/code/proposal/validator.ex index cbad3bb50..7f5d57e0d 100644 --- a/lib/archethic/governance/code/proposal/validator.ex +++ b/lib/archethic/governance/code/proposal/validator.ex @@ -6,7 +6,6 @@ defmodule Archethic.Governance.Code.Proposal.Validator do """ require Logger - alias Archethic.Utils alias Archethic.Utils.Regression @marker Application.compile_env(:archethic, :marker) @@ -20,7 +19,7 @@ defmodule Archethic.Governance.Code.Proposal.Validator do _ <- Process.sleep(3 * 60 * 1000), :ok <- Regression.run_benchmarks(nodes, phase: :after), :ok <- Regression.run_playbooks(nodes), - {:ok, metrics} <- + {:ok, _metrics} <- Regression.get_metrics("collector", 9090, 5 + System.monotonic_time(:second) - start) do write_metrics(metrics) end From 67140f5ed93d66ff9d8f3053c1e994aee0511c28 Mon Sep 17 00:00:00 2001 From: tenmoves Date: Thu, 13 Apr 2023 15:29:42 +0000 Subject: [PATCH 11/21] fix governance validation workflow --- .iex.exs | 24 ------------ Dockerfile | 2 +- lib/archethic/crypto.ex | 36 ++++++++---------- lib/archethic/db/embedded_impl/chain_index.ex | 2 +- .../governance/code/cicd/docker/cicd.ex | 30 ++++++++++++--- .../governance/code/proposal/validator.ex | 18 +++++---- .../ip_lookup/nat_discovery/miniupnp.ex | 2 +- lib/archethic/utils/testnet.ex | 37 +++++++++++++++---- lib/mix/tasks/validate.ex | 15 +++----- scripts/governance/proposal_ci_job.sh | 3 -- src/c/nat/miniupnp | 2 +- 11 files changed, 92 insertions(+), 79 deletions(-) diff --git a/.iex.exs b/.iex.exs index 9cb865592..6dbc4a11a 100644 --- a/.iex.exs +++ b/.iex.exs @@ -13,27 +13,3 @@ alias Archethic.TransactionChain alias Archethic.TransactionChain.Transaction alias Archethic.TransactionChain.TransactionData alias Archethic.BeaconChain - - -alias Archethic.Governance.Code.CICD.Docker -prop = %Archethic.Governance.Code.Proposal{ - address: <<0, 0, 135, 189, 175, 8, 43, 160, 49, 21, 23, 241, 61, 189, 221, - 177, 45, 120, 59, 243, 80, 75, 193, 250, 119, 188, 219, 73, 209, 197, 118, - 170, 122, 142>>, - previous_public_key: <<0, 1, 62, 205, 23, 142, 29, 217, 94, 202, 136, 109, - 165, 135, 153, 205, 202, 132, 122, 116, 97, 18, 90, 34, 141, 225, 172, 138, - 16, 138, 85, 15, 41, 149>>, - timestamp: nil, - description: "\Testing code proposal\"", - changes: "diff --git a/mix.exs b/mix.exs\nindex a82c0b3c..b94d4323 100644\n--- a/mix.exs\n+++ b/mix.exs\n@@ -4,7 +4,7 @@ defmodule Archethic.MixProject do\n def project do\n [\n app: :archethic,\n- version: \"1.0.7\",\n+ version: \"1.0.8\",\n build_path: \"_build\",\n config_path: \"config/config.exs\",\n deps_path: \"deps\",\ndiff --git a/rel/appups/archethic/1.0.7_to_1.0.8.appup b/rel/appups/archethic/1.0.7_to_1.0.8.appup\nnew file mode 100644\nindex 00000000..18b6d541\n--- /dev/null\n+++ b/rel/appups/archethic/1.0.7_to_1.0.8.appup\n@@ -0,0 +1,4 @@\n+{\"1.0.8\",\n+ [{\"1.0.7\",\n+ [{load_module,'Elixir.Archethic', []}]}],\n+ [{\"1.0.7\",\n+ [{load_module,'Elixir.Archethic', []}]}]\n+}.\n", - version: "1.0.8", - files: ["mix.exs", "rel/appups/archethic/1.0.7_to_1.0.8.appup"], - approvals: [ - <<0, 0, 135, 189, 175, 8, 43, 160, 49, 21, 23, 241, 61, 189, 221, 177, 45, - 120, 59, 243, 80, 75, 193, 250, 119, 188, 219, 73, 209, 197, 118, 170, - 122, 142>>, - <<0, 0, 168, 220, 209, 92, 167, 255, 187, 168, 176, 63, 128, 210, 199, 35, - 63, 0, 252, 36, 147, 196, 130, 249, 233, 17, 132, 0, 6, 88, 232, 198, 98, - 44>> - ] -} diff --git a/Dockerfile b/Dockerfile index e07eff4bf..7ba5288a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,7 +62,7 @@ RUN git config user.name aebot \ # build release RUN mix assets.deploy -RUN MIX_ENV=prod mix distillery.release +RUN MIX_ENV=${MIX_ENV} mix distillery.release # Install RUN mkdir -p /opt/app \ && tar zxf /opt/code/_build/${MIX_ENV}/rel/archethic_node/releases/*/archethic_node.tar.gz -C /opt/app diff --git a/lib/archethic/crypto.ex b/lib/archethic/crypto.ex index 21df8ab28..041f88678 100755 --- a/lib/archethic/crypto.ex +++ b/lib/archethic/crypto.ex @@ -1141,26 +1141,22 @@ defmodule Archethic.Crypto do """ @spec get_root_ca_public_key(key()) :: binary() def get_root_ca_public_key(<>) do - if System.get_env("ARCHETHIC_NETWORK_TYPE") == "testnet" do - "" - else - case Keyword.get(@certification_public_keys, ID.to_origin(origin_id)) do - nil -> - "no_key" - - # Only for dev - [] -> - "" - - curves when is_list(curves) -> - case Keyword.get(curves, ID.to_curve(curve)) do - nil -> - "no_key" - - public_key -> - public_key - end - end + case Keyword.get(@certification_public_keys, ID.to_origin(origin_id)) do + nil -> + "no_key" + + # Only for dev + [] -> + "" + + curves when is_list(curves) -> + case Keyword.get(curves, ID.to_curve(curve)) do + nil -> + "no_key" + + public_key -> + public_key + end end end diff --git a/lib/archethic/db/embedded_impl/chain_index.ex b/lib/archethic/db/embedded_impl/chain_index.ex index e451036a8..c7d4a397b 100644 --- a/lib/archethic/db/embedded_impl/chain_index.ex +++ b/lib/archethic/db/embedded_impl/chain_index.ex @@ -40,7 +40,7 @@ defmodule Archethic.DB.EmbeddedImpl.ChainIndex do {:ok, %{db_path: db_path}} end - def code_change("1.0.6", state, _extra) do + def code_change("1.0.7", state, _extra) do ## We start the unstarted LRU cache cache_max_size = Application.get_env(:archethic, Archethic.DB.ChainIndex.MaxCacheSize) LRU.start_link(@archetic_db_tx_index_cache, cache_max_size) diff --git a/lib/archethic/governance/code/cicd/docker/cicd.ex b/lib/archethic/governance/code/cicd/docker/cicd.ex index 1dbb274e3..23d31448c 100644 --- a/lib/archethic/governance/code/cicd/docker/cicd.ex +++ b/lib/archethic/governance/code/cicd/docker/cicd.ex @@ -216,7 +216,8 @@ defmodule Archethic.Governance.Code.CICD.Docker do |> Path.basename() |> String.downcase() - validator_container = "#{compose_prefix}-validator-1" + validator_1_container = "#{compose_prefix}-validator_1-1" + validator_2_container = "#{compose_prefix}-validator_2-1" nodes = 1..nb_nodes |> Enum.map(&"#{compose_prefix}-node#{&1}-1") @@ -226,13 +227,13 @@ defmodule Archethic.Governance.Code.CICD.Docker do %{cmd: {_, 0}, testnet: _testnet} <- testnet_start(dir, nb_nodes), # wait until the validator is ready for upgrade :ok <- Logger.info("#{dir} Part I", address: address_encoded), - {:ok, _} <- wait_for_marker(validator_container, @marker), + {:ok, _} <- wait_for_marker(validator_1_container, @marker), :ok <- Logger.info("#{dir} Upgrade", address: address_encoded), true <- testnet_upgrade(dir, nodes, version), :ok <- Logger.info("#{dir} Part II", address: address_encoded), - # {_, 0} <- validator_continue(validator_container, testnet), + {_, 0} <- validator_continue(dir), 0 <- - docker_wait(validator_container, System.monotonic_time(:second)) do + docker_wait(validator_2_container, System.monotonic_time(:second)) do testnet_cleanup(dir, 0, address_encoded) else _ -> @@ -240,11 +241,13 @@ defmodule Archethic.Governance.Code.CICD.Docker do end end + @logfile_name "ci_logfile.txt" defp testnet_prepare(dir, address, version) do ci = container_name(address) with :ok <- File.mkdir_p!(dir), - {_, 0} <- docker(["cp", "#{ci}:#{@releases}/#{version}/#{@release}", dir]) do + {_, 0} <- docker(["cp", "#{ci}:#{@releases}/#{version}/#{@release}", dir]), + {_, 0} <- docker(["cp", "#{ci}:/opt/code/#{@logfile_name}", dir <> "/#{@logfile_name}"]) do :ok else _ -> :error @@ -340,6 +343,23 @@ defmodule Archethic.Governance.Code.CICD.Docker do end end + defp validator_continue(dir) do + compose = compose_file(dir) + + System.cmd( + "docker-compose", + [ + "--profile", + "validate_2", + "-f", + compose, + "up", + "-d" + ], + @cmd_options + ) + end + defp testnet_cleanup(dir, code, address_encoded) do Logger.info("#{dir} Cleanup", address: address_encoded) diff --git a/lib/archethic/governance/code/proposal/validator.ex b/lib/archethic/governance/code/proposal/validator.ex index 7f5d57e0d..d04a5e7da 100644 --- a/lib/archethic/governance/code/proposal/validator.ex +++ b/lib/archethic/governance/code/proposal/validator.ex @@ -7,19 +7,23 @@ defmodule Archethic.Governance.Code.Proposal.Validator do require Logger alias Archethic.Utils.Regression + alias Archethic.Utils @marker Application.compile_env(:archethic, :marker) - def run(nodes, _) do + def run(nodes, 1) do + with true <- Regression.nodes_up?(nodes), + :ok <- Regression.run_benchmarks(nodes, phase: :before) do + put_marker(nodes) + end + end + + def run(nodes, 2) do start = System.monotonic_time(:second) - with true <- Regression.nodes_up?(nodes), - :ok <- Regression.run_benchmarks(nodes, phase: :before), - :ok <- put_marker(nodes), - _ <- Process.sleep(3 * 60 * 1000), - :ok <- Regression.run_benchmarks(nodes, phase: :after), + with :ok <- Regression.run_benchmarks(nodes, phase: :after), :ok <- Regression.run_playbooks(nodes), - {:ok, _metrics} <- + {:ok, metrics} <- Regression.get_metrics("collector", 9090, 5 + System.monotonic_time(:second) - start) do write_metrics(metrics) end diff --git a/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex b/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex index 07b2a7ac1..6e803c761 100644 --- a/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex +++ b/lib/archethic/networking/ip_lookup/nat_discovery/miniupnp.ex @@ -3,7 +3,7 @@ defmodule Archethic.Networking.IPLookup.NATDiscovery.MiniUPNP do require Logger - def upnpc() do + defp upnpc() do Application.app_dir(:archethic, "priv/c_dist/upnpc") end diff --git a/lib/archethic/utils/testnet.ex b/lib/archethic/utils/testnet.ex index 83c2b8199..858b95779 100644 --- a/lib/archethic/utils/testnet.ex +++ b/lib/archethic/utils/testnet.ex @@ -112,7 +112,8 @@ defmodule Archethic.Utils.Testnet do defp p2p_port, do: 30_002 defp web_port, do: 40_000 - @validator_ip 220 + @validator_1_ip 220 + @validator_2_ip 230 @bench_ip 221 @collector_ip 200 @@ -280,7 +281,7 @@ defmodule Archethic.Utils.Testnet do networks: %{:net => %{ipv4_address: "1.2.3.#{@collector_ip}"}}, volumes: [".prometheus.yml:/etc/prometheus/prometheus.yml:ro"] }, - "validator" => %{ + "validator_1" => %{ image: "i", environment: %{ "ARCHETHIC_MUT_DIR" => "/opt/data" @@ -289,7 +290,19 @@ defmodule Archethic.Utils.Testnet do volumes: [ "./validator_data/:/opt/data" ], - networks: %{:net => %{ipv4_address: "1.2.3.#{@validator_ip}"}}, + networks: %{:net => %{ipv4_address: "1.2.3.#{@validator_1_ip}"}}, + }, + "validator_2" => %{ + image: "i", + environment: %{ + "ARCHETHIC_MUT_DIR" => "/opt/data" + }, + command: ["./bin/archethic_node", "validate", "1.2.3.2", "1.2.3.3", "1.2.3.4", "--phase=2"], + volumes: [ + "./validator_data/:/opt/data" + ], + networks: %{:net => %{ipv4_address: "1.2.3.#{@validator_2_ip}"}}, + profiles: ["validate_2"] }, "bench" => %{ image: "i", @@ -330,7 +343,7 @@ defmodule Archethic.Utils.Testnet do networks: %{:net => %{ipv4_address: ip.(@collector_ip)}}, volumes: [".prometheus.yml:/etc/prometheus/prometheus.yml:ro"] }) - |> Map.put("validator", %{ + |> Map.put("validator_1", %{ image: image, environment: %{ "ARCHETHIC_MUT_DIR" => "/opt/data" @@ -339,8 +352,19 @@ defmodule Archethic.Utils.Testnet do volumes: [ "./validator_data/:/opt/data" ], - # profiles: ["validate"], - networks: %{:net => %{ipv4_address: ip.(@validator_ip)}} + networks: %{:net => %{ipv4_address: ip.(@validator_1_ip)}} + }) + |> Map.put("validator_2", %{ + image: image, + environment: %{ + "ARCHETHIC_MUT_DIR" => "/opt/data" + }, + command: ["./bin/archethic_node", "validate" | uninodes] ++ ["--phase=2"], + volumes: [ + "./validator_data/:/opt/data" + ], + profiles: ["validate_2"], + networks: %{:net => %{ipv4_address: ip.(@validator_2_ip)}} }) |> Map.put("bench", %{ image: image, @@ -351,7 +375,6 @@ defmodule Archethic.Utils.Testnet do volumes: [ "./bench_data/:/opt/data" ], - # profiles: ["validate"], networks: %{:net => %{ipv4_address: ip.(@bench_ip)}} }) diff --git a/lib/mix/tasks/validate.ex b/lib/mix/tasks/validate.ex index a34d60da1..d917b2956 100644 --- a/lib/mix/tasks/validate.ex +++ b/lib/mix/tasks/validate.ex @@ -20,6 +20,7 @@ defmodule Mix.Tasks.Archethic.Proposal.Validator do use Mix.Task alias Archethic.Governance.Code.Proposal.Validator + alias Mix.Tasks.Utils @impl Mix.Task @spec run([binary]) :: any @@ -37,16 +38,12 @@ defmodule Mix.Tasks.Archethic.Proposal.Validator do if parsed[:help] do Mix.shell().cmd("mix help #{Mix.Task.task_name(__MODULE__)}") else - :ok = maybe(parsed, :phase, &Validator.run/2, [nodes, parsed[:phase]]) + :ok = + Utils.apply_function_if_key_exists(parsed, :phase, &Validator.run/2, [ + nodes, + parsed[:phase] + ]) end end end - - defp maybe(opts, key, func, args) do - if opts[key] do - apply(func, args) - else - :ok - end - end end diff --git a/scripts/governance/proposal_ci_job.sh b/scripts/governance/proposal_ci_job.sh index 6f1504b9b..abf948267 100755 --- a/scripts/governance/proposal_ci_job.sh +++ b/scripts/governance/proposal_ci_job.sh @@ -9,9 +9,6 @@ PROPOSAL_ADDRESS=$1 PROPOSAL_DESCRIPTION=$2 PROPOSAL_FILENAME=./proposal.diff -echo "=== Test proposal ${PROPOSAL_ADDRESS}" -tee ${PROPOSAL_FILENAME} - echo "=== Create branch ${PROPOSAL_ADDRESS}" git checkout -b "prop_${PROPOSAL_ADDRESS}" diff --git a/src/c/nat/miniupnp b/src/c/nat/miniupnp index 836fba468..214deacb1 160000 --- a/src/c/nat/miniupnp +++ b/src/c/nat/miniupnp @@ -1 +1 @@ -Subproject commit 836fba4684578cf3ca190d21c63e0383e0a4d3b7 +Subproject commit 214deacb11aa9176f00eaac65450c8c9a5a6026c From e69dbf15e1bacfbe2b7ec655825fed658d03bca1 Mon Sep 17 00:00:00 2001 From: tenmoves Date: Thu, 11 May 2023 20:35:42 +0100 Subject: [PATCH 12/21] put back P2P message benchmark --- config/config.exs | 6 +++--- lib/archethic/utils/regression.ex | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/config/config.exs b/config/config.exs index c3ac66f6e..8c883d538 100644 --- a/config/config.exs +++ b/config/config.exs @@ -12,9 +12,9 @@ config :git_hooks, {:cmd, "mix credo"}, {:cmd, "mix sobelow"}, {:cmd, "mix knigge.verify"}, - # {:cmd, "mix test --trace"}, - {:cmd, "mix dialyzer"} - # {:cmd, "mix check.updates"} + {:cmd, "mix test --trace"}, + {:cmd, "mix dialyzer"}, + {:cmd, "mix check.updates"} ] ] ] diff --git a/lib/archethic/utils/regression.ex b/lib/archethic/utils/regression.ex index aac018a6a..489f46652 100644 --- a/lib/archethic/utils/regression.ex +++ b/lib/archethic/utils/regression.ex @@ -10,9 +10,10 @@ defmodule Archethic.Utils.Regression do alias Archethic.Utils.WebClient alias Archethic.Utils.Regression.Benchmark.EndToEndValidation + alias Archethic.Utils.Regression.Benchmark.P2PMessage @playbooks [UCO] - @benchmarks [EndToEndValidation] + @benchmarks [P2PMessage, EndToEndValidation] def run_playbooks(nodes, opts \\ []) do Logger.debug("Running playbooks on #{inspect(nodes)} with #{inspect(opts)}") From 4eda088ac3f54a734aaec870f32320c0e1b7c662 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Thu, 25 May 2023 09:40:31 +0200 Subject: [PATCH 13/21] Use only 3 nodes for the CD to reduce memory usage --- lib/archethic/governance/code/cicd/docker/cicd.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/archethic/governance/code/cicd/docker/cicd.ex b/lib/archethic/governance/code/cicd/docker/cicd.ex index 23d31448c..6e1ced9d7 100644 --- a/lib/archethic/governance/code/cicd/docker/cicd.ex +++ b/lib/archethic/governance/code/cicd/docker/cicd.ex @@ -209,7 +209,7 @@ defmodule Archethic.Governance.Code.CICD.Docker do Logger.info("Running proposal", address: address_encoded) dir = temp_dir("utn-#{address_encoded}-") - nb_nodes = 5 + nb_nodes = 3 compose_prefix = dir From 2a1a86af818e603e5c2b2b1561a65112a7b87599 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Thu, 25 May 2023 09:40:59 +0200 Subject: [PATCH 14/21] Fix containers names --- lib/archethic/governance/code/cicd/docker/cicd.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/archethic/governance/code/cicd/docker/cicd.ex b/lib/archethic/governance/code/cicd/docker/cicd.ex index 6e1ced9d7..499962a85 100644 --- a/lib/archethic/governance/code/cicd/docker/cicd.ex +++ b/lib/archethic/governance/code/cicd/docker/cicd.ex @@ -216,10 +216,10 @@ defmodule Archethic.Governance.Code.CICD.Docker do |> Path.basename() |> String.downcase() - validator_1_container = "#{compose_prefix}-validator_1-1" - validator_2_container = "#{compose_prefix}-validator_2-1" + validator_1_container = "#{compose_prefix}_validator_1_1" + validator_2_container = "#{compose_prefix}_validator_2_1" - nodes = 1..nb_nodes |> Enum.map(&"#{compose_prefix}-node#{&1}-1") + nodes = 1..nb_nodes |> Enum.map(&"#{compose_prefix}_node#{&1}_1") with :ok <- Logger.info("#{dir} Prepare", address: address_encoded), :ok <- testnet_prepare(dir, address, version), From ea992b10eb5f04e66ebdc7a354b9210cfdbb1e95 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Thu, 25 May 2023 09:44:00 +0200 Subject: [PATCH 15/21] bigger timeout for the upgrade part --- lib/archethic/governance/code/cicd/docker/cicd.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/archethic/governance/code/cicd/docker/cicd.ex b/lib/archethic/governance/code/cicd/docker/cicd.ex index 499962a85..541e98772 100644 --- a/lib/archethic/governance/code/cicd/docker/cicd.ex +++ b/lib/archethic/governance/code/cicd/docker/cicd.ex @@ -296,7 +296,7 @@ defmodule Archethic.Governance.Code.CICD.Docker do raise "Upgrade failed" end end, - timeout: 30_000, + timeout: 120_000, ordered: false ) |> Enum.into([]) From e8117cb0f434b475a8446c13974202621247e792 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Thu, 25 May 2023 09:44:30 +0200 Subject: [PATCH 16/21] remove tons of warning and debugs --- lib/archethic/governance/code/cicd/docker/cicd.ex | 2 -- lib/mix/tasks/validate.ex | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/archethic/governance/code/cicd/docker/cicd.ex b/lib/archethic/governance/code/cicd/docker/cicd.ex index 541e98772..5972af9be 100644 --- a/lib/archethic/governance/code/cicd/docker/cicd.ex +++ b/lib/archethic/governance/code/cicd/docker/cicd.ex @@ -329,8 +329,6 @@ defmodule Archethic.Governance.Code.CICD.Docker do defp wait_for_marker_loop(port, marker) do receive do {^port, {:data, {:eol, line}}} -> - Logger.debug(line) - if String.starts_with?(line, marker) do line else diff --git a/lib/mix/tasks/validate.ex b/lib/mix/tasks/validate.ex index d917b2956..e76f31b15 100644 --- a/lib/mix/tasks/validate.ex +++ b/lib/mix/tasks/validate.ex @@ -25,6 +25,8 @@ defmodule Mix.Tasks.Archethic.Proposal.Validator do @impl Mix.Task @spec run([binary]) :: any def run(args) do + Application.ensure_all_started(:telemetry) + case OptionParser.parse!(args, strict: [ help: :boolean, From dbf0680efbad91c0902a3fd95630e4eb8f172856 Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Thu, 25 May 2023 09:45:46 +0200 Subject: [PATCH 17/21] log the error --- lib/archethic/governance/code/cicd/docker/cicd.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/archethic/governance/code/cicd/docker/cicd.ex b/lib/archethic/governance/code/cicd/docker/cicd.ex index 5972af9be..8234e6f05 100644 --- a/lib/archethic/governance/code/cicd/docker/cicd.ex +++ b/lib/archethic/governance/code/cicd/docker/cicd.ex @@ -236,7 +236,8 @@ defmodule Archethic.Governance.Code.CICD.Docker do docker_wait(validator_2_container, System.monotonic_time(:second)) do testnet_cleanup(dir, 0, address_encoded) else - _ -> + err -> + Logger.error("CD FAILED reason: #{inspect(err)}") testnet_cleanup(dir, 1, address_encoded) end end From fb5d9b878d5a428584215a2134c4a06327ca0c0d Mon Sep 17 00:00:00 2001 From: Bastien CHAMAGNE Date: Thu, 25 May 2023 09:58:40 +0200 Subject: [PATCH 18/21] bind the log file to the host /tmp folder --- lib/archethic/governance/code/cicd/docker/cicd.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/archethic/governance/code/cicd/docker/cicd.ex b/lib/archethic/governance/code/cicd/docker/cicd.ex index 8234e6f05..1e923c785 100644 --- a/lib/archethic/governance/code/cicd/docker/cicd.ex +++ b/lib/archethic/governance/code/cicd/docker/cicd.ex @@ -143,14 +143,20 @@ defmodule Archethic.Governance.Code.CICD.Docker do :ok end + @logfile_name "ci_logfile.txt" @ci_script "/opt/code/scripts/governance/proposal_ci_job.sh" defp do_run_docker_ci(%Proposal{address: address, changes: changes, description: description}) do Logger.info("Verify proposal", address: Base.encode16(address)) name = container_name(address) + System.cmd("rm", ["-f", "/tmp/#{@logfile_name}"]) + System.cmd("touch", ["/tmp/#{@logfile_name}"]) + args = [ "run", + "-v", + "/tmp/#{@logfile_name}:/opt/code/#{@logfile_name}", "--entrypoint", @ci_script, "-i", @@ -242,7 +248,6 @@ defmodule Archethic.Governance.Code.CICD.Docker do end end - @logfile_name "ci_logfile.txt" defp testnet_prepare(dir, address, version) do ci = container_name(address) From b0df5134a069244a4bfb1e34cb025ab37ca843c0 Mon Sep 17 00:00:00 2001 From: Samuel Manzanera Date: Fri, 26 May 2023 13:42:48 +0200 Subject: [PATCH 19/21] Leverage CI on code approval to trigger CD on governance pipeline --- lib/archethic/governance.ex | 23 +++++++++++++++---- lib/archethic/governance/code.ex | 38 ++++++++++++++------------------ 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/lib/archethic/governance.ex b/lib/archethic/governance.ex index 44570740e..a13b573f0 100644 --- a/lib/archethic/governance.ex +++ b/lib/archethic/governance.ex @@ -5,15 +5,21 @@ defmodule Archethic.Governance do """ alias Archethic.Crypto + alias Archethic.Election alias __MODULE__.Code alias __MODULE__.Code.Proposal alias __MODULE__.Pools + alias Archethic.P2P + alias Archethic.TransactionChain alias Archethic.TransactionChain.Transaction alias Archethic.TransactionChain.TransactionData + alias Archethic.TaskSupervisor + alias Archethic.Utils + @proposal_tx_select_fields [ :address, :timestamp, @@ -105,11 +111,20 @@ defmodule Archethic.Governance do recipients: [prop_address] } }) do - if Code.testnet_deployment?(prop_address) do - {:ok, prop} = get_code_proposal(prop_address) - Code.deploy_proposal_testnet(prop) + + storage_nodes = Election.chain_storage_nodes(prop_address, P2P.authorized_and_available_nodes()) + + with true <- Utils.key_in_node_list?(storage_nodes, Crypto.first_node_public_key()), + {:ok, prop} <- get_code_proposal(prop_address), + true <- Code.enough_code_approval?(prop) do + Task.Supervisor.start_child(TaskSupervisor, fn -> + if Code.valid_integration?(prop) do + Code.deploy_proposal_testnet(prop) + end + end) else - :ok + _ -> + :ok end end diff --git a/lib/archethic/governance/code.ex b/lib/archethic/governance/code.ex index 6f2f95cd0..bd087c989 100644 --- a/lib/archethic/governance/code.ex +++ b/lib/archethic/governance/code.ex @@ -3,21 +3,11 @@ defmodule Archethic.Governance.Code do Provide functions to handle the code management and deployment """ - alias Archethic.Crypto - - alias Archethic.Election - alias __MODULE__.CICD alias __MODULE__.Proposal alias Archethic.Governance.Pools - alias Archethic.P2P - - alias Archethic.TransactionChain - - alias Archethic.Utils - @src_dir Application.compile_env(:archethic, :src_dir) @doc """ @@ -41,17 +31,23 @@ defmodule Archethic.Governance.Code do @doc """ Determine if the code proposal can be deployed into testnet """ - @spec testnet_deployment?(binary()) :: boolean() - def testnet_deployment?(proposal_address) when is_binary(proposal_address) do - storage_nodes = - Election.chain_storage_nodes(proposal_address, P2P.authorized_and_available_nodes()) - - if Utils.key_in_node_list?(storage_nodes, Crypto.first_node_public_key()) do - approvals = TransactionChain.list_signatures_for_pending_transaction(proposal_address) - ratio = length(approvals) / length(Pools.members_of(:technical_council)) - ratio >= Pools.threshold_acceptance_for(:technical_council) - else - false + @spec enough_code_approval?(Proposal.t()) :: boolean() + def enough_code_approval?(%Proposal{approvals: approvals}) do + ratio = length(approvals) / length(Pools.members_of(:technical_council)) + ratio >= Pools.threshold_acceptance_for(:technical_council) + end + + @doc """ + Determines if the CI passes for the given proposal + """ + @spec valid_integration?(Proposal.t()) :: boolean() + def valid_integration?(prop = %Proposal{}) do + try do + CICD.run_ci!(prop) + true + rescue + _ -> + false end end From f2eebfc56c246c4dbbeb1b8d3ccd7d66c06b0888 Mon Sep 17 00:00:00 2001 From: Samuel Manzanera Date: Fri, 26 May 2023 13:43:52 +0200 Subject: [PATCH 20/21] Remove CI container before to start --- lib/archethic/governance/code/cicd/docker/cicd.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/archethic/governance/code/cicd/docker/cicd.ex b/lib/archethic/governance/code/cicd/docker/cicd.ex index 1e923c785..baabaf83d 100644 --- a/lib/archethic/governance/code/cicd/docker/cicd.ex +++ b/lib/archethic/governance/code/cicd/docker/cicd.ex @@ -153,6 +153,8 @@ defmodule Archethic.Governance.Code.CICD.Docker do System.cmd("rm", ["-f", "/tmp/#{@logfile_name}"]) System.cmd("touch", ["/tmp/#{@logfile_name}"]) + System.cmd(System.find_executable("docker"), ["rm", name]) + args = [ "run", "-v", From 92aa7938f4e79009b6f7eaf61acfaf44d5661ef3 Mon Sep 17 00:00:00 2001 From: Samuel Manzanera Date: Fri, 26 May 2023 16:00:32 +0200 Subject: [PATCH 21/21] Fix format --- lib/archethic/governance.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/archethic/governance.ex b/lib/archethic/governance.ex index a13b573f0..5e3cdf818 100644 --- a/lib/archethic/governance.ex +++ b/lib/archethic/governance.ex @@ -111,8 +111,8 @@ defmodule Archethic.Governance do recipients: [prop_address] } }) do - - storage_nodes = Election.chain_storage_nodes(prop_address, P2P.authorized_and_available_nodes()) + storage_nodes = + Election.chain_storage_nodes(prop_address, P2P.authorized_and_available_nodes()) with true <- Utils.key_in_node_list?(storage_nodes, Crypto.first_node_public_key()), {:ok, prop} <- get_code_proposal(prop_address),