diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8aefa25ae..b18f0eead 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,72 +2,71 @@ name: Elixir CI on: push: - branches: [ master, develop, testnet ] + branches: [master, develop, testnet] pull_request: - branches: [ master, develop, testnet ] + branches: [master, develop, testnet] env: MIX_ENV: test jobs: build: - name: Build and test runs-on: ubuntu-20.04 - + steps: - - name: Install OS Packages - uses: mstksg/get-package@2a4b48d55d72d43ca89ae58ec9ca1397d34a1c35 - with: - apt-get: libgmp-dev libssl-dev libtinfo-dev libsystemd-dev zlib1g-dev libsodium-dev autoconf-archive libcmocka0 libcmocka-dev procps iproute2 build-essential git pkg-config gcc libtool automake libssl-dev uthash-dev autoconf doxygen libjson-c-dev libini-config-dev libcurl4-openssl-dev libltdl-dev libtss2-dev tss2 - - uses: actions/checkout@v2 - - name: Set up Elixir - uses: erlef/setup-beam@988e02bfe678367a02564f65ca2e37726dc0268f - id: beam - with: - elixir-version: '1.14.1' # Define the elixir version [required] - otp-version: '25.1' # Define the OTP version [required] - - name: Restore dependencies cache - uses: actions/cache@v3 - id: mix-cache - with: - path: deps - key: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-mix-${{ hashFiles('**/mix.lock') }} - restore-keys: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-mix- - - name: Make Clean - run: mix clean - - name: Install dependencies - if: steps.mix-cache.outputs.cache-hit != 'true' - run: mix deps.get - - name: Check dependency updates - run: mix hex.outdated --within-requirements 1>/dev/null || echo 'Updates available!' - - name: Set Formatting - run: mix format --check-formatted - - name: Restore build cache - uses: actions/cache@v3 - with: - path: _build - key: ${{ runner.os }}-build-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('**/mix.lock') }} - restore-keys: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-build- - - name: Compile the codebase - run: mix compile --warnings-as-errors - - name: Set credo - run: mix credo - - name: Run Sobelow - run: mix sobelow - - name: Run tests - run: mix test --trace - - name: Retrieve PLT Cache - uses: actions/cache@v3 - id: plt-cache - with: - path: priv/plts - key: ${{ runner.os }}-plts-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('**/mix.lock') }} - restore-keys: ${{ runner.os }}-plts-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}- - - name: Create PLTs - if: steps.plt-cache.outputs.cache-hit != 'true' - run: | - mkdir -p priv/plts - mix dialyzer --plt - - name: Run dialyzer - run: mix dialyzer --no-check --ignore-exit-status + - name: Install OS Packages + uses: mstksg/get-package@2a4b48d55d72d43ca89ae58ec9ca1397d34a1c35 + with: + apt-get: libgmp-dev libssl-dev libtinfo-dev libsystemd-dev zlib1g-dev libsodium-dev autoconf-archive libcmocka0 libcmocka-dev procps iproute2 build-essential git pkg-config gcc libtool automake libssl-dev uthash-dev autoconf doxygen libjson-c-dev libini-config-dev libcurl4-openssl-dev libltdl-dev libtss2-dev tss2 + - uses: actions/checkout@v2 + - name: Set up Elixir + uses: erlef/setup-beam@988e02bfe678367a02564f65ca2e37726dc0268f + id: beam + with: + elixir-version: "1.14.1" # Define the elixir version [required] + otp-version: "25.1" # Define the OTP version [required] + - name: Restore dependencies cache + uses: actions/cache@v3 + id: mix-cache + with: + path: deps + key: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-mix-${{ hashFiles('**/mix.lock') }} + restore-keys: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-mix- + - name: Make Clean + run: mix clean + - name: Install dependencies + if: steps.mix-cache.outputs.cache-hit != 'true' + run: mix deps.get + - name: Check dependency updates + run: mix hex.outdated --within-requirements 1>/dev/null || echo 'Updates available!' + - name: Set Formatting + run: mix format --check-formatted + - name: Restore build cache + uses: actions/cache@v3 + with: + path: _build + key: ${{ runner.os }}-build-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('**/mix.lock') }} + restore-keys: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-build- + - name: Compile the codebase + run: mix compile --warnings-as-errors + - name: Set credo + run: mix credo + - name: Run Sobelow + run: mix sobelow + - name: Run tests + run: mix test --trace + - name: Retrieve PLT Cache + uses: actions/cache@v3 + id: plt-cache + with: + path: priv/plts + key: ${{ runner.os }}-plts-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('**/mix.lock') }} + restore-keys: ${{ runner.os }}-plts-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}- + - name: Create PLTs + if: steps.plt-cache.outputs.cache-hit != 'true' + run: | + mkdir -p priv/plts + mix dialyzer --plt + - name: Run dialyzer + run: mix dialyzer --no-check --ignore-exit-status diff --git a/Dockerfile b/Dockerfile index 928b188dc..7ba5288a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,9 @@ FROM elixir:1.14.1-alpine AS archethic-ci -ARG skip_tests=0 +ARG with_tests=1 ARG MIX_ENV=prod +ARG USER_ID +ARG GROUP_ID # CI # - compile @@ -19,25 +21,22 @@ ARG MIX_ENV=prod # - code # - release -# running TESTNET with release upgrade should ??? +ENV ARCHETHIC_NETWORK_TYPE=testnet RUN apk add --no-cache --update \ build-base \ + grep \ bash \ gcc \ make \ g++ \ - libexecinfo-dev \ - libexecinfo \ git \ npm \ - python3 \ wget \ openssl \ libsodium-dev \ - gmp-dev \ - miniupnpc - + libexecinfo-dev \ + gmp-dev # Install hex and rebar RUN mix local.rebar --force \ @@ -61,23 +60,12 @@ RUN git config user.name aebot \ && git config user.email aebot@archethic.net \ && git remote add origin https://github.com/archethic-foundation/archethic-node -# Install Dart Sass -RUN npm install -g sass - -# build Sass -> CSS -RUN cd assets && \ - sass --no-source-map --style=compressed css/app.scss ../priv/static/css/app.css && cd - - # build release -RUN mix do assets.deploy, distillery.release - -# gen PLT -RUN if [ $with_tests -eq 1 ]; then mix git_hooks.run pre_push ;fi - +RUN mix assets.deploy +RUN MIX_ENV=${MIX_ENV} mix distillery.release # Install RUN mkdir -p /opt/app \ - && cd /opt/app \ - && tar zxf /opt/code/_build/${MIX_ENV}/rel/archethic_node/releases/*/archethic_node.tar.gz + && tar zxf /opt/code/_build/${MIX_ENV}/rel/archethic_node/releases/*/archethic_node.tar.gz -C /opt/app CMD /opt/app/bin/archethic_node foreground ################################################################################ @@ -86,7 +74,10 @@ FROM archethic-ci as build FROM elixir:1.14.1-alpine -RUN apk add --no-cache --update bash git openssl libsodium +ARG USER_ID +ARG GROUP_ID + +RUN apk add --no-cache --update bash git openssl libsodium libexecinfo COPY --from=build /opt/app /opt/app COPY --from=build /opt/code/.git /opt/code/.git @@ -94,5 +85,8 @@ COPY --from=build /opt/code/.git /opt/code/.git WORKDIR /opt/code RUN git reset --hard +RUN rm -rf /opt/code/.git +RUN rm -rf /opt/code/priv + WORKDIR /opt/app CMD /opt/app/bin/archethic_node foreground diff --git a/Makefile b/Makefile index 29a30534a..61d79bf1a 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,6 @@ ifeq ($(TPM_INSTALLED),0) $(CC) src/c/crypto/tpm/keygen.c src/c/crypto/tpm/lib.c -o priv/c_dist/tpm_keygen -I src/c/crypto/tpm/lib.h $(TPMFLAGS) endif - clean: rm -f priv/c_dist/* mix archethic.clean_db diff --git a/assets/package-lock.json b/assets/package-lock.json index 42a8b78ff..0709b3d89 100644 --- a/assets/package-lock.json +++ b/assets/package-lock.json @@ -24,13 +24,15 @@ } }, "../deps/phoenix": { - "version": "0.0.1" + "version": "1.6.15", + "license": "MIT" }, "../deps/phoenix_html": { - "version": "0.0.1" + "version": "3.2.0" }, "../deps/phoenix_live_view": { - "version": "0.0.1" + "version": "0.18.11", + "license": "MIT" }, "node_modules/@ampproject/remapping": { "version": "2.2.0", diff --git a/config/dev.exs b/config/dev.exs index f50c59bf7..2169b4973 100755 --- a/config/dev.exs +++ b/config/dev.exs @@ -167,13 +167,13 @@ config :archethic, ArchethicWeb.Endpoint, config :archethic, :throttle, by_ip_high: [ period: 1000, - limit: 5_000 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_HIGH", "5000") ], by_ip_low: [ period: 1000, - limit: 5_000 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_LOW", "5000") ], by_ip_and_path: [ period: 1000, - limit: 5_000 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_AND_PATH", "5000") ] diff --git a/config/prod.exs b/config/prod.exs index 828205dd4..7d0940864 100755 --- a/config/prod.exs +++ b/config/prod.exs @@ -89,14 +89,21 @@ config :archethic, Archethic.BeaconChain.SummaryTimer, config :archethic, Archethic.Crypto, root_ca_public_keys: [ - software: [ - secp256r1: - System.get_env( - "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY", - "04F0FE701A03CE375A6E57ADBE0255808812036571C1424DB2779C77E8B4A9BA80A15B118E8E7465EE2E94094E59C4B3F7177E99063AF1B19BFCC4D7E1AC3F89DD" - ) - |> Base.decode16!(case: :mixed) - ], + software: + case System.get_env("ARCHETHIC_NETWORK_TYPE") do + "testnet" -> + [] + + _ -> + [ + secp256r1: + System.get_env( + "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY", + "04F0FE701A03CE375A6E57ADBE0255808812036571C1424DB2779C77E8B4A9BA80A15B118E8E7465EE2E94094E59C4B3F7177E99063AF1B19BFCC4D7E1AC3F89DD" + ) + |> Base.decode16!(case: :mixed) + ] + end, tpm: [ secp256r1: System.get_env( @@ -273,13 +280,13 @@ config :archethic, ArchethicWeb.Endpoint, config :archethic, :throttle, by_ip_high: [ period: 1000, - limit: 500 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_HIGH", "500") ], by_ip_low: [ period: 1000, - limit: 20 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_LOW", "20") ], by_ip_and_path: [ period: 1000, - limit: 20 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_AND_PATH", "20") ] diff --git a/config/test.exs b/config/test.exs index 1c323e56a..d5b81db1f 100755 --- a/config/test.exs +++ b/config/test.exs @@ -179,13 +179,13 @@ config :archethic, ArchethicWeb.Endpoint, config :archethic, :throttle, by_ip_high: [ period: 1000, - limit: 5_000 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_HIGH", "5000") ], by_ip_low: [ period: 1000, - limit: 5_000 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_LOW", "5000") ], by_ip_and_path: [ period: 1000, - limit: 5_000 + limit: System.get_env("ARCHETHIC_THROTTLE_IP_AND_PATH", "5000") ] diff --git a/lib/archethic/governance.ex b/lib/archethic/governance.ex index adf25659e..f951ca39f 100644 --- a/lib/archethic/governance.ex +++ b/lib/archethic/governance.ex @@ -5,15 +5,21 @@ defmodule Archethic.Governance do """ alias Archethic.Crypto + alias Archethic.Election alias __MODULE__.Code alias __MODULE__.Code.Proposal alias __MODULE__.Pools + alias Archethic.P2P + alias Archethic.TransactionChain alias Archethic.TransactionChain.Transaction alias Archethic.TransactionChain.TransactionData + alias Archethic.TaskSupervisor + alias Archethic.Utils + @proposal_tx_select_fields [ :address, :timestamp, @@ -105,11 +111,20 @@ defmodule Archethic.Governance do recipients: [prop_address] } }) do - if Code.testnet_deployment?(prop_address) do - {:ok, prop} = get_code_proposal(prop_address) - Code.deploy_proposal_testnet(prop) + storage_nodes = + Election.chain_storage_nodes(prop_address, P2P.authorized_and_available_nodes()) + + with true <- Utils.key_in_node_list?(storage_nodes, Crypto.first_node_public_key()), + {:ok, prop} <- get_code_proposal(prop_address), + true <- Code.enough_code_approval?(prop) do + Task.Supervisor.start_child(TaskSupervisor, fn -> + if Code.valid_integration?(prop) do + Code.deploy_proposal_testnet(prop) + end + end) else - :ok + _ -> + :ok end end diff --git a/lib/archethic/governance/code.ex b/lib/archethic/governance/code.ex index b821bee2c..bd087c989 100644 --- a/lib/archethic/governance/code.ex +++ b/lib/archethic/governance/code.ex @@ -3,21 +3,11 @@ defmodule Archethic.Governance.Code do Provide functions to handle the code management and deployment """ - alias Archethic.Crypto - - alias Archethic.Election - alias __MODULE__.CICD alias __MODULE__.Proposal alias Archethic.Governance.Pools - alias Archethic.P2P - - alias Archethic.TransactionChain - - alias Archethic.Utils - @src_dir Application.compile_env(:archethic, :src_dir) @doc """ @@ -41,17 +31,23 @@ defmodule Archethic.Governance.Code do @doc """ Determine if the code proposal can be deployed into testnet """ - @spec testnet_deployment?(binary()) :: boolean() - def testnet_deployment?(proposal_address) when is_binary(proposal_address) do - storage_nodes = - Election.chain_storage_nodes(proposal_address, P2P.authorized_and_available_nodes()) - - if Utils.key_in_node_list?(storage_nodes, Crypto.first_node_public_key()) do - approvals = TransactionChain.get_signatures_for_pending_transaction(proposal_address) - ratio = length(approvals) / length(Pools.members_of(:technical_council)) - ratio >= Pools.threshold_acceptance_for(:technical_council) - else - false + @spec enough_code_approval?(Proposal.t()) :: boolean() + def enough_code_approval?(%Proposal{approvals: approvals}) do + ratio = length(approvals) / length(Pools.members_of(:technical_council)) + ratio >= Pools.threshold_acceptance_for(:technical_council) + end + + @doc """ + Determines if the CI passes for the given proposal + """ + @spec valid_integration?(Proposal.t()) :: boolean() + def valid_integration?(prop = %Proposal{}) do + try do + CICD.run_ci!(prop) + true + rescue + _ -> + false end end diff --git a/lib/archethic/governance/code/cicd/docker/cicd.ex b/lib/archethic/governance/code/cicd/docker/cicd.ex index 9b7bb33f2..baabaf83d 100644 --- a/lib/archethic/governance/code/cicd/docker/cicd.ex +++ b/lib/archethic/governance/code/cicd/docker/cicd.ex @@ -123,18 +123,42 @@ defmodule Archethic.Governance.Code.CICD.Docker do # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ [CI] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ defp build_ci_image do - {_, 0} = docker(["build", "-t", "archethic-ci", "--target", "archethic-ci", "."]) + {user_id, _} = System.cmd("id", ["-u"]) + {group_id, _} = System.cmd("id", ["-g"]) + + {_, 0} = + docker([ + "build", + "-t", + "archethic-ci", + "--target", + "archethic-ci", + "--build-arg", + "USER_ID=#{String.trim(user_id)}", + "--build-arg", + "GROUP_ID=#{String.trim(group_id)}", + "." + ]) + :ok end + @logfile_name "ci_logfile.txt" @ci_script "/opt/code/scripts/governance/proposal_ci_job.sh" defp do_run_docker_ci(%Proposal{address: address, changes: changes, description: description}) do Logger.info("Verify proposal", address: Base.encode16(address)) name = container_name(address) + System.cmd("rm", ["-f", "/tmp/#{@logfile_name}"]) + System.cmd("touch", ["/tmp/#{@logfile_name}"]) + + System.cmd(System.find_executable("docker"), ["rm", name]) + args = [ "run", + "-v", + "/tmp/#{@logfile_name}:/opt/code/#{@logfile_name}", "--entrypoint", @ci_script, "-i", @@ -166,63 +190,98 @@ defmodule Archethic.Governance.Code.CICD.Docker do # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ [CD] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ defp build_cd_image do - {_, 0} = docker(["build", "-t", "archethic-cd", "."]) - :ok - end - - @releases "/opt/code/_build/dev/rel/archethic_node/releases" - @release "archethic_node.tar.gz" - - defp testnet_prepare(dir, address, version) do - ci = container_name(address) + {user_id, _} = System.cmd("id", ["-u"]) + {group_id, _} = System.cmd("id", ["-g"]) + + {_, 0} = + docker([ + "build", + "-t", + "archethic-cd", + "--build-arg", + "USER_ID=#{String.trim(user_id)}", + "--build-arg", + "GROUP_ID=#{String.trim(group_id)}", + "." + ]) - with :ok <- File.mkdir_p!(dir), - {_, 0} <- docker(["cp", "#{ci}:#{@releases}/#{version}/#{@release}", dir]) do - :ok - else - _ -> :error - end + :ok end @marker Application.compile_env(:archethic, :marker) + @releases "/opt/code/_build/prod/rel/archethic_node/releases" + @release "archethic_node.tar.gz" defp do_run_docker_testnet(%Proposal{address: address, version: version}) do address_encoded = Base.encode16(address) Logger.info("Running proposal", address: address_encoded) dir = temp_dir("utn-#{address_encoded}-") - nb_nodes = 5 + nb_nodes = 3 + + compose_prefix = + dir + |> Path.basename() + |> String.downcase() - compose_prefix = Path.basename(dir) - validator_container = "#{compose_prefix}_validator_1" - validator_continue = ["ash", "-c", "echo 'yes' > /proc/1/fd/0"] + validator_1_container = "#{compose_prefix}_validator_1_1" + validator_2_container = "#{compose_prefix}_validator_2_1" nodes = 1..nb_nodes |> Enum.map(&"#{compose_prefix}_node#{&1}_1") with :ok <- Logger.info("#{dir} Prepare", address: address_encoded), :ok <- testnet_prepare(dir, address, version), :ok <- Logger.info("#{dir} Start", address: address_encoded), - {_, 0} <- testnet_start(dir, nb_nodes), + %{cmd: {_, 0}, testnet: _testnet} <- testnet_start(dir, nb_nodes), # wait until the validator is ready for upgrade :ok <- Logger.info("#{dir} Part I", address: address_encoded), - {:ok, _} <- wait_for_marker(validator_container, @marker), + {:ok, _} <- wait_for_marker(validator_1_container, @marker), :ok <- Logger.info("#{dir} Upgrade", address: address_encoded), true <- testnet_upgrade(dir, nodes, version), :ok <- Logger.info("#{dir} Part II", address: address_encoded), - {_, 0} <- docker_exec(validator_container, validator_continue), - 0 <- docker_wait(validator_container, System.monotonic_time(:second)) do + {_, 0} <- validator_continue(dir), + 0 <- + docker_wait(validator_2_container, System.monotonic_time(:second)) do testnet_cleanup(dir, 0, address_encoded) else - _ -> + err -> + Logger.error("CD FAILED reason: #{inspect(err)}") testnet_cleanup(dir, 1, address_encoded) end end - defp testnet_cleanup(dir, code, address_encoded) do - Logger.info("#{dir} Cleanup", address: address_encoded) - System.cmd("docker-compose", ["-f", compose_file(dir), "down"], @cmd_options) - File.rm_rf!(dir) - code + defp testnet_prepare(dir, address, version) do + ci = container_name(address) + + with :ok <- File.mkdir_p!(dir), + {_, 0} <- docker(["cp", "#{ci}:#{@releases}/#{version}/#{@release}", dir]), + {_, 0} <- docker(["cp", "#{ci}:/opt/code/#{@logfile_name}", dir <> "/#{@logfile_name}"]) do + :ok + else + _ -> :error + end + end + + @subnet "172.16.100.0/24" + + defp testnet_start(dir, nb_nodes) do + compose = compose_file(dir) + options = [image: "archethic-cd", dir: dir, src: @src_dir, persist: false] + + Stream.iterate(@subnet, &Subnet.next/1) + |> Stream.take(123) + |> Stream.map(fn subnet -> + testnet = Testnet.from(nb_nodes, Keyword.put(options, :subnet, subnet)) + + with :ok <- Testnet.create!(testnet, dir) do + %{ + testnet: testnet, + cmd: System.cmd("docker-compose", ["-f", compose, "up", "-d"], @cmd_options) + } + end + end) + |> Stream.filter(&(elem(&1[:cmd], 1) == 0)) + |> Enum.at(0) end defp testnet_upgrade(dir, containers, version) do @@ -245,7 +304,7 @@ defmodule Archethic.Governance.Code.CICD.Docker do raise "Upgrade failed" end end, - timeout: 30_000, + timeout: 120_000, ordered: false ) |> Enum.into([]) @@ -255,26 +314,7 @@ defmodule Archethic.Governance.Code.CICD.Docker do result end - @subnet "172.16.100.0/24" - - defp testnet_start(dir, nb_nodes) do - compose = compose_file(dir) - options = [image: "archethic-cd", dir: dir, src: @src_dir, persist: false] - - Stream.iterate(@subnet, &Subnet.next/1) - |> Stream.take(123) - |> Stream.map(fn subnet -> - testnet = Testnet.from(nb_nodes, Keyword.put(options, :subnet, subnet)) - - with :ok <- Testnet.create!(testnet, dir) do - System.cmd("docker-compose", ["-f", compose, "up", "-d"], @cmd_options) - end - end) - |> Stream.filter(&(elem(&1, 1) == 0)) - |> Enum.at(0) - end - - defp wait_for_marker(container_name, marker, timeout \\ 600_000) do + defp wait_for_marker(container_name, marker, timeout \\ 60_000) do args = ["logs", container_name, "--follow", "--tail", "10"] opts = [:binary, :use_stdio, :stderr_to_stdout, line: 8192, args: args] @@ -297,8 +337,6 @@ defmodule Archethic.Governance.Code.CICD.Docker do defp wait_for_marker_loop(port, marker) do receive do {^port, {:data, {:eol, line}}} -> - Logger.debug(line) - if String.starts_with?(line, marker) do line else @@ -311,9 +349,51 @@ defmodule Archethic.Governance.Code.CICD.Docker do end end - defp compose_file(dir), do: Path.join(dir, "docker-compose.json") + defp validator_continue(dir) do + compose = compose_file(dir) - defp docker_exec(container_name, cmd), do: docker(["exec", container_name] ++ cmd) + System.cmd( + "docker-compose", + [ + "--profile", + "validate_2", + "-f", + compose, + "up", + "-d" + ], + @cmd_options + ) + end + + defp testnet_cleanup(dir, code, address_encoded) do + Logger.info("#{dir} Cleanup", address: address_encoded) + + System.cmd( + "docker-compose", + [ + "-f", + compose_file(dir), + "down", + "--volumes" + ], + @cmd_options + ) + + docker([ + "image", + "rm", + "-f", + "archethic-cd", + "archethic-ci", + "prom/prometheus" + ]) + + File.rm_rf!(dir) + code + end + + defp compose_file(dir), do: Path.join(dir, "docker-compose.json") defp temp_dir(prefix, tmp \\ System.tmp_dir!()) do {_mega, sec, micro} = :os.timestamp() diff --git a/lib/archethic/governance/code/proposal/validator.ex b/lib/archethic/governance/code/proposal/validator.ex index 6d4d5a4f0..d04a5e7da 100644 --- a/lib/archethic/governance/code/proposal/validator.ex +++ b/lib/archethic/governance/code/proposal/validator.ex @@ -6,28 +6,31 @@ defmodule Archethic.Governance.Code.Proposal.Validator do """ require Logger - alias Archethic.Utils alias Archethic.Utils.Regression + alias Archethic.Utils @marker Application.compile_env(:archethic, :marker) - def run(nodes) do + def run(nodes, 1) do + with true <- Regression.nodes_up?(nodes), + :ok <- Regression.run_benchmarks(nodes, phase: :before) do + put_marker(nodes) + end + end + + def run(nodes, 2) do start = System.monotonic_time(:second) - with true <- Regression.nodes_up?(nodes), - :ok <- Regression.run_benchmarks(nodes, phase: :before), - :ok <- await_upgrade(nodes), - :ok <- Regression.run_benchmarks(nodes, phase: :after), + with :ok <- Regression.run_benchmarks(nodes, phase: :after), :ok <- Regression.run_playbooks(nodes), {:ok, metrics} <- Regression.get_metrics("collector", 9090, 5 + System.monotonic_time(:second) - start) do - File.write!(Utils.mut_dir("metrics"), :erlang.term_to_iovec(metrics)) + write_metrics(metrics) end end - defp await_upgrade(args) do - IO.puts("#{@marker} | #{inspect(args)}") - Port.open({:fd, 0, 1}, [:out, {:line, 256}]) |> Port.command("") - IO.gets("Continue? ") |> IO.puts() - end + defp write_metrics(metrics), + do: File.write!(Utils.mut_dir("metrics"), :erlang.term_to_iovec(metrics)) + + defp put_marker(args), do: IO.puts("#{@marker} | #{inspect(args)}") end diff --git a/lib/archethic/networking/port_forwarding.ex b/lib/archethic/networking/port_forwarding.ex index 0460dbb42..097431f96 100644 --- a/lib/archethic/networking/port_forwarding.ex +++ b/lib/archethic/networking/port_forwarding.ex @@ -62,7 +62,7 @@ defmodule Archethic.Networking.PortForwarding do {:ok, port} :error -> - Logger.warning("Cannot publish the a random port #{port}") + Logger.warning("Cannot publish the random port #{port}") fallback(port, _force? = true, retries - 1) end diff --git a/lib/archethic/p2p/listener.ex b/lib/archethic/p2p/listener.ex index ff2f8b5fe..67f1d8767 100644 --- a/lib/archethic/p2p/listener.ex +++ b/lib/archethic/p2p/listener.ex @@ -17,9 +17,9 @@ defmodule Archethic.P2P.Listener do def init(opts) do transport = Keyword.get(opts, :transport) port = Keyword.get(opts, :port) - PubSub.register_to_node_status() + # test(transport, port) {:ok, %{transport: transport, port: port}} end diff --git a/lib/archethic/utils/regression.ex b/lib/archethic/utils/regression.ex index e148d1e2b..489f46652 100644 --- a/lib/archethic/utils/regression.ex +++ b/lib/archethic/utils/regression.ex @@ -6,14 +6,13 @@ defmodule Archethic.Utils.Regression do alias Archethic.Utils - alias Archethic.Utils.Regression.Playbook.SmartContract alias Archethic.Utils.Regression.Playbook.UCO alias Archethic.Utils.WebClient alias Archethic.Utils.Regression.Benchmark.EndToEndValidation alias Archethic.Utils.Regression.Benchmark.P2PMessage - @playbooks [UCO, SmartContract] + @playbooks [UCO] @benchmarks [P2PMessage, EndToEndValidation] def run_playbooks(nodes, opts \\ []) do @@ -81,10 +80,15 @@ defmodule Archethic.Utils.Regression do |> Enum.all?(&(&1 == {:ok, :ok})) end - defp node_up?(node, start \\ System.monotonic_time(:millisecond), timeout \\ 5 * 60_000) + def node_up?(node, start \\ System.monotonic_time(:millisecond), timeout \\ 5 * 60_000) - defp node_up?(node, start, timeout) do - port = Application.get_env(:archethic, ArchethicWeb.Endpoint)[:http][:port] + def node_up?(node, start, timeout) do + port = + if System.get_env("ARCHETHIC_NETWORK_TYPE") == "testnet" do + 40_000 + else + Application.get_env(:archethic, ArchethicWeb.Endpoint)[:http][:port] + end case WebClient.with_connection(node, port, &WebClient.request(&1, "GET", "/up")) do {:ok, ["up"]} -> diff --git a/lib/archethic/utils/testnet.ex b/lib/archethic/utils/testnet.ex index 523e66ef4..858b95779 100644 --- a/lib/archethic/utils/testnet.ex +++ b/lib/archethic/utils/testnet.ex @@ -112,7 +112,8 @@ defmodule Archethic.Utils.Testnet do defp p2p_port, do: 30_002 defp web_port, do: 40_000 - @validator_ip 220 + @validator_1_ip 220 + @validator_2_ip 230 @bench_ip 221 @collector_ip 200 @@ -140,19 +141,19 @@ defmodule Archethic.Utils.Testnet do "- job_name: testnet\\n" <> " static_configs:\\n" <> " - targets:\\n" <> - " - node1:40000\\n" <> - " - node2:40000\\n" <> - " - node3:40000\\n\\n"}, + " - 1.2.3.2:40000\\n" <> + " - 1.2.3.3:40000\\n" <> + " - 1.2.3.4:40000\\n\\n"}, {"docker-compose.json", %{ version: "3.9", networks: %{:net => %{ipam: %{config: [%{subnet: "1.2.3.0/24"}], driver: :default}}}, services: %{ "node1" => %{ + build: %{context: "c"}, environment: %{ "ARCHETHIC_CRYPTO_SEED" => "node1", "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => "1.2.3.2:30002:00011D967D71B2E135C84206DDD108B5925A2CD99C8EBC5AB5D8FD2EC9400CE3C98A:tcp", "ARCHETHIC_STATIC_IP" => "1.2.3.2", - "ARCHETHIC_DB_HOST" => "scylladb1:9042", "ARCHETHIC_NETWORKING_IMPL" => "STATIC", "ARCHETHIC_NETWORKING_PORT_FORWARDING" => "false", "ARCHETHIC_NODE_ALLOWED_KEY_ORIGINS" => "software", @@ -165,17 +166,17 @@ defmodule Archethic.Utils.Testnet do "ARCHETHIC_SHARED_SECRETS_APPLICATION_INTERVAL" => "0 * * * * * *", "ARCHETHIC_SELF_REPAIR_SCHEDULER_INTRERVAL" => "5 * * * * * *", "ARCHETHIC_NODE_IP_VALIDATION" => "false", - "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE" + "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE", + "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY" => "", + "ARCHETHIC_CRYPTO_ROOT_CA_TPM_PUBKEY" => "", + "ARCHETHIC_NETWORK_TYPE" => "testnet", + "ARCHETHIC_THROTTLE_IP_AND_PATH" => 999999, + "ARCHETHIC_THROTTLE_IP_HIGH" => 999999, + "ARCHETHIC_THROTTLE_IP_LOW" => 999999 }, image: "i", - build: %{ context: "c"}, networks: %{:net => %{ipv4_address: "1.2.3.2"}}, command: [ - "/wait-for-tcp.sh", - "scylladb1:9042", - "--timeout=0", - "--strict", - "--", "./bin/archethic_node", "foreground" ], @@ -185,11 +186,11 @@ defmodule Archethic.Utils.Testnet do }, "node2" => %{ depends_on: ["node1"], + build: %{context: "c"}, environment: %{ "ARCHETHIC_CRYPTO_SEED" => "node2", "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => "1.2.3.2:30002:00011D967D71B2E135C84206DDD108B5925A2CD99C8EBC5AB5D8FD2EC9400CE3C98A:tcp", "ARCHETHIC_STATIC_IP" => "1.2.3.3", - "ARCHETHIC_DB_HOST" => "scylladb2:9042", "ARCHETHIC_NETWORKING_IMPL" => "STATIC", "ARCHETHIC_NETWORKING_PORT_FORWARDING" => "false", "ARCHETHIC_NODE_ALLOWED_KEY_ORIGINS" => "software", @@ -202,7 +203,13 @@ defmodule Archethic.Utils.Testnet do "ARCHETHIC_SHARED_SECRETS_APPLICATION_INTERVAL" => "0 * * * * * *", "ARCHETHIC_SELF_REPAIR_SCHEDULER_INTRERVAL" => "5 * * * * * *", "ARCHETHIC_NODE_IP_VALIDATION" => "false", - "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE" + "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE", + "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY" => "", + "ARCHETHIC_CRYPTO_ROOT_CA_TPM_PUBKEY" => "", + "ARCHETHIC_NETWORK_TYPE" => "testnet", + "ARCHETHIC_THROTTLE_IP_AND_PATH" => 999999, + "ARCHETHIC_THROTTLE_IP_HIGH" => 999999, + "ARCHETHIC_THROTTLE_IP_LOW" => 999999 }, image: "i", networks: %{:net => %{ipv4_address: "1.2.3.3"}}, @@ -212,26 +219,24 @@ defmodule Archethic.Utils.Testnet do ], command: [ "/wait-for-tcp.sh", - "scylladb2:9042", - "--timeout=0", - "--strict", "--", - "/wait-for-tcp.sh", - "node1:40000", + "--host=1.2.3.2", + "--port=30002", "--timeout=0", - "--strict", "--", + "--strict", + "--", "/wait-for-node.sh", - "http://node1:40000/up", + "http://1.2.3.2:40000/up", "./bin/archethic_node", "foreground" ] }, "node3" => %{ depends_on: ["node1"], + build: %{context: "c"}, environment: %{ "ARCHETHIC_CRYPTO_SEED" => "node3", "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => "1.2.3.2:30002:00011D967D71B2E135C84206DDD108B5925A2CD99C8EBC5AB5D8FD2EC9400CE3C98A:tcp", "ARCHETHIC_STATIC_IP" => "1.2.3.4", - "ARCHETHIC_DB_HOST" => "scylladb3:9042", "ARCHETHIC_NETWORKING_IMPL" => "STATIC", "ARCHETHIC_NETWORKING_PORT_FORWARDING" => "false", "ARCHETHIC_NODE_ALLOWED_KEY_ORIGINS" => "software", @@ -244,7 +249,13 @@ defmodule Archethic.Utils.Testnet do "ARCHETHIC_SHARED_SECRETS_APPLICATION_INTERVAL" => "0 * * * * * *", "ARCHETHIC_SELF_REPAIR_SCHEDULER_INTRERVAL" => "5 * * * * * *", "ARCHETHIC_NODE_IP_VALIDATION" => "false", - "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE" + "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE", + "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY" => "", + "ARCHETHIC_CRYPTO_ROOT_CA_TPM_PUBKEY" => "", + "ARCHETHIC_NETWORK_TYPE" => "testnet", + "ARCHETHIC_THROTTLE_IP_AND_PATH" => 999999, + "ARCHETHIC_THROTTLE_IP_HIGH" => 999999, + "ARCHETHIC_THROTTLE_IP_LOW" => 999999 }, image: "i", networks: %{:net => %{ipv4_address: "1.2.3.4"}}, @@ -254,15 +265,13 @@ defmodule Archethic.Utils.Testnet do ], command: [ "/wait-for-tcp.sh", - "scylladb3:9042", - "--timeout=0", - "--strict", "--", - "/wait-for-tcp.sh", - "node1:40000", + "--host=1.2.3.2", + "--port=30002", "--timeout=0", - "--strict", "--", + "--strict", + "--", "/wait-for-node.sh", - "http://node1:40000/up", + "http://1.2.3.2:40000/up", "./bin/archethic_node", "foreground" ] @@ -272,41 +281,39 @@ defmodule Archethic.Utils.Testnet do networks: %{:net => %{ipv4_address: "1.2.3.#{@collector_ip}"}}, volumes: [".prometheus.yml:/etc/prometheus/prometheus.yml:ro"] }, - "validator" => %{ - image: "archethic-node:latest", + "validator_1" => %{ + image: "i", environment: %{ "ARCHETHIC_MUT_DIR" => "/opt/data" }, - command: ["./bin/archethic_node", "regression_test", "--validate", "node1", "node2", "node3"], + command: ["./bin/archethic_node", "validate", "1.2.3.2", "1.2.3.3", "1.2.3.4", "--phase=1"], volumes: [ "./validator_data/:/opt/data" ], - profiles: ["validate"], - networks: %{:net => %{ipv4_address: "1.2.3.#{@validator_ip}"}}, + networks: %{:net => %{ipv4_address: "1.2.3.#{@validator_1_ip}"}}, + }, + "validator_2" => %{ + image: "i", + environment: %{ + "ARCHETHIC_MUT_DIR" => "/opt/data" + }, + command: ["./bin/archethic_node", "validate", "1.2.3.2", "1.2.3.3", "1.2.3.4", "--phase=2"], + volumes: [ + "./validator_data/:/opt/data" + ], + networks: %{:net => %{ipv4_address: "1.2.3.#{@validator_2_ip}"}}, + profiles: ["validate_2"] }, "bench" => %{ - image: "archethic-node:latest", + image: "i", environment: %{ "ARCHETHIC_MUT_DIR" => "/opt/data" }, - command: ["./bin/archethic_node", "regression_test", "--bench", "node1", "node2", "node3"], + command: ["./bin/archethic_node", "regression_test", "--bench", "1.2.3.2", "1.2.3.3", "1.2.3.4"], volumes: [ "./bench_data/:/opt/data" ], - profiles: ["validate"], networks: %{ :net => %{ipv4_address: "1.2.3.#{@bench_ip}"}} - }, - "scylladb1" => %{ - image: "scylladb/scylla", - networks: %{:net => %{ipv4_address: "1.2.3.51"}} - }, - "scylladb2" => %{ - image: "scylladb/scylla", - networks: %{:net => %{ipv4_address: "1.2.3.52"}} - }, - "scylladb3" => %{ - image: "scylladb/scylla", - networks: %{:net => %{ipv4_address: "1.2.3.53"}} } } } @@ -323,18 +330,11 @@ defmodule Archethic.Utils.Testnet do ip = fn i -> Subnet.at(base, i) end services = nodes_from(nb_nodes, src, image, ip) - uninodes = Map.keys(services) - networks = %{:net => %{ipam: %{driver: :default, config: [%{subnet: subnet}]}}} + uninodes = + Enum.map(services, fn {_, %{environment: %{"ARCHETHIC_STATIC_IP" => ip}}} -> ip end) - databases = - 1..nb_nodes - |> Enum.reduce(%{}, fn i, acc -> - Map.put(acc, "scylladb#{i}", %{ - image: "scylladb/scylla", - networks: %{:net => %{ipv4_address: ip.(i + 50)}} - }) - end) + networks = %{:net => %{ipam: %{driver: :default, config: [%{subnet: subnet}]}}} services = services @@ -343,20 +343,31 @@ defmodule Archethic.Utils.Testnet do networks: %{:net => %{ipv4_address: ip.(@collector_ip)}}, volumes: [".prometheus.yml:/etc/prometheus/prometheus.yml:ro"] }) - |> Map.put("validator", %{ - image: "archethic-node:latest", + |> Map.put("validator_1", %{ + image: image, + environment: %{ + "ARCHETHIC_MUT_DIR" => "/opt/data" + }, + command: ["./bin/archethic_node", "validate" | uninodes] ++ ["--phase=1"], + volumes: [ + "./validator_data/:/opt/data" + ], + networks: %{:net => %{ipv4_address: ip.(@validator_1_ip)}} + }) + |> Map.put("validator_2", %{ + image: image, environment: %{ "ARCHETHIC_MUT_DIR" => "/opt/data" }, - command: ["./bin/archethic_node", "regression_test", "--validate" | uninodes], + command: ["./bin/archethic_node", "validate" | uninodes] ++ ["--phase=2"], volumes: [ "./validator_data/:/opt/data" ], - profiles: ["validate"], - networks: %{:net => %{ipv4_address: ip.(@validator_ip)}} + profiles: ["validate_2"], + networks: %{:net => %{ipv4_address: ip.(@validator_2_ip)}} }) |> Map.put("bench", %{ - image: "archethic-node:latest", + image: image, environment: %{ "ARCHETHIC_MUT_DIR" => "/opt/data" }, @@ -364,10 +375,8 @@ defmodule Archethic.Utils.Testnet do volumes: [ "./bench_data/:/opt/data" ], - profiles: ["validate"], networks: %{:net => %{ipv4_address: ip.(@bench_ip)}} }) - |> Map.merge(databases) compose = %{version: "3.9", services: services, networks: networks} @@ -419,6 +428,8 @@ defmodule Archethic.Utils.Testnet do end defp to_node(1, src, image, ip) do + node_1_ip_address = ip.(1 + 1) + {"node1", %{ build: %{context: src}, @@ -426,9 +437,8 @@ defmodule Archethic.Utils.Testnet do environment: %{ "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE", "ARCHETHIC_CRYPTO_SEED" => "node1", - "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => seeder(ip.(1 + 1)), + "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => seeder(node_1_ip_address), "ARCHETHIC_STATIC_IP" => ip.(1 + 1), - "ARCHETHIC_DB_HOST" => "scylladb1:9042", "ARCHETHIC_NETWORKING_IMPL" => "STATIC", "ARCHETHIC_NETWORKING_PORT_FORWARDING" => "false", "ARCHETHIC_NODE_ALLOWED_KEY_ORIGINS" => "software", @@ -440,18 +450,19 @@ defmodule Archethic.Utils.Testnet do "ARCHETHIC_SHARED_SECRETS_RENEWAL_SCHEDULER_INTERVAL" => "40 * * * * * *", "ARCHETHIC_SHARED_SECRETS_APPLICATION_INTERVAL" => "0 * * * * * *", "ARCHETHIC_SELF_REPAIR_SCHEDULER_INTRERVAL" => "5 * * * * * *", - "ARCHETHIC_NODE_IP_VALIDATION" => "false" + "ARCHETHIC_NODE_IP_VALIDATION" => "false", + "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY" => "", + "ARCHETHIC_CRYPTO_ROOT_CA_TPM_PUBKEY" => "", + "ARCHETHIC_NETWORK_TYPE" => "testnet", + "ARCHETHIC_THROTTLE_IP_HIGH" => 999_999, + "ARCHETHIC_THROTTLE_IP_LOW" => 999_999, + "ARCHETHIC_THROTTLE_IP_AND_PATH" => 999_999 }, volumes: [ "#{Path.join([src, "/scripts/wait-for-tcp.sh"])}:/wait-for-tcp.sh:ro" ], networks: %{:net => %{ipv4_address: ip.(1 + 1)}}, command: [ - "/wait-for-tcp.sh", - "scylladb1:9042", - "--timeout=0", - "--strict", - "--", "./bin/archethic_node", "foreground" ] @@ -459,16 +470,19 @@ defmodule Archethic.Utils.Testnet do end defp to_node(n, src, image, ip) do + node_1_ip_address = ip.(1 + 1) + ip_address = ip.(n + 1) + {"node#{n}", %{ + build: %{context: src}, image: image, depends_on: ["node1"], environment: %{ "ARCHETHIC_CRYPTO_NODE_KEYSTORE_IMPL" => "SOFTWARE", "ARCHETHIC_CRYPTO_SEED" => "node#{n}", - "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => seeder(ip.(1 + 1)), - "ARCHETHIC_STATIC_IP" => ip.(n + 1), - "ARCHETHIC_DB_HOST" => "scylladb#{n}:9042", + "ARCHETHIC_P2P_BOOTSTRAPPING_SEEDS" => seeder(node_1_ip_address), + "ARCHETHIC_STATIC_IP" => ip_address, "ARCHETHIC_NETWORKING_IMPL" => "STATIC", "ARCHETHIC_NETWORKING_PORT_FORWARDING" => "false", "ARCHETHIC_NODE_ALLOWED_KEY_ORIGINS" => "software", @@ -480,26 +494,28 @@ defmodule Archethic.Utils.Testnet do "ARCHETHIC_SHARED_SECRETS_RENEWAL_SCHEDULER_INTERVAL" => "40 * * * * * *", "ARCHETHIC_SHARED_SECRETS_APPLICATION_INTERVAL" => "0 * * * * * *", "ARCHETHIC_SELF_REPAIR_SCHEDULER_INTRERVAL" => "5 * * * * * *", - "ARCHETHIC_NODE_IP_VALIDATION" => "false" + "ARCHETHIC_NODE_IP_VALIDATION" => "false", + "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY" => "", + "ARCHETHIC_CRYPTO_ROOT_CA_TPM_PUBKEY" => "", + "ARCHETHIC_NETWORK_TYPE" => "testnet", + "ARCHETHIC_THROTTLE_IP_HIGH" => 999_999, + "ARCHETHIC_THROTTLE_IP_LOW" => 999_999, + "ARCHETHIC_THROTTLE_IP_AND_PATH" => 999_999 }, - networks: %{:net => %{ipv4_address: ip.(n + 1)}}, + networks: %{:net => %{ipv4_address: ip_address}}, volumes: [ "#{Path.join([src, "/scripts/wait-for-tcp.sh"])}:/wait-for-tcp.sh:ro", "#{Path.join([src, "/scripts/wait-for-node.sh"])}:/wait-for-node.sh:ro" ], command: [ "/wait-for-tcp.sh", - "scylladb#{n}:9042", - "--timeout=0", - "--strict", - "--", - "/wait-for-tcp.sh", - "node1:40000", + "--host=#{node_1_ip_address}", + "--port=#{p2p_port()}", "--timeout=0", "--strict", "--", "/wait-for-node.sh", - "http://node1:#{web_port()}/up", + "http://#{node_1_ip_address}:#{web_port()}/up", "./bin/archethic_node", "foreground" ] diff --git a/lib/archethic_web/plugs/plug_throttle_by_ip_and_path.ex b/lib/archethic_web/plugs/plug_throttle_by_ip_and_path.ex index 237b38c08..429e23625 100644 --- a/lib/archethic_web/plugs/plug_throttle_by_ip_and_path.ex +++ b/lib/archethic_web/plugs/plug_throttle_by_ip_and_path.ex @@ -10,6 +10,8 @@ defmodule ArchethicWeb.PlugThrottleByIPandPath do rule "Throttle by IP and Path", conn do [period: period, limit: limit] = Application.get_env(:archethic, :throttle)[:by_ip_and_path] + limit = String.to_integer(limit) + throttle({conn.remote_ip, conn.path_info}, period: period, limit: limit, diff --git a/lib/archethic_web/plugs/plug_throttle_by_ip_high.ex b/lib/archethic_web/plugs/plug_throttle_by_ip_high.ex index fb3083ab4..2dc9f5551 100644 --- a/lib/archethic_web/plugs/plug_throttle_by_ip_high.ex +++ b/lib/archethic_web/plugs/plug_throttle_by_ip_high.ex @@ -9,6 +9,8 @@ defmodule ArchethicWeb.PlugThrottleByIPHigh do rule "Throttle by IP", conn do [period: period, limit: limit] = Application.get_env(:archethic, :throttle)[:by_ip_high] + limit = String.to_integer(limit) + throttle(conn.remote_ip, period: period, limit: limit, diff --git a/lib/archethic_web/plugs/plug_throttle_by_ip_low.ex b/lib/archethic_web/plugs/plug_throttle_by_ip_low.ex index 9a38210c3..ca3e9f61f 100644 --- a/lib/archethic_web/plugs/plug_throttle_by_ip_low.ex +++ b/lib/archethic_web/plugs/plug_throttle_by_ip_low.ex @@ -9,6 +9,8 @@ defmodule ArchethicWeb.PlugThrottleByIPLow do rule "Throttle by IP", conn do [period: period, limit: limit] = Application.get_env(:archethic, :throttle)[:by_ip_low] + limit = String.to_integer(limit) + throttle(conn.remote_ip, period: period, limit: limit, diff --git a/lib/mix/tasks/regression.ex b/lib/mix/tasks/regression.ex index 67867fa26..69b1002bf 100644 --- a/lib/mix/tasks/regression.ex +++ b/lib/mix/tasks/regression.ex @@ -23,6 +23,7 @@ defmodule Mix.Tasks.Archethic.Regression do use Mix.Task alias Archethic.Utils.Regression + alias Mix.Tasks.Utils @impl Mix.Task def run(args) do @@ -43,17 +44,17 @@ defmodule Mix.Tasks.Archethic.Regression do Mix.shell().cmd("mix help #{Mix.Task.task_name(__MODULE__)}") else true = Regression.nodes_up?(nodes) - :ok = maybe(parsed, :bench, &Regression.run_benchmarks/1, [nodes]) - :ok = maybe(parsed, :playbook, &Regression.run_playbooks/1, [nodes]) - end - end - end - defp maybe(opts, key, func, args) do - if opts[key] do - apply(func, args) - else - :ok + :ok = + Utils.apply_function_if_key_exists(parsed, :bench, &Regression.run_benchmarks/1, [ + nodes + ]) + + :ok = + Utils.apply_function_if_key_exists(parsed, :playbook, &Regression.run_playbooks/1, [ + nodes + ]) + end end end end diff --git a/lib/mix/tasks/utils.ex b/lib/mix/tasks/utils.ex new file mode 100644 index 000000000..5601f1f88 --- /dev/null +++ b/lib/mix/tasks/utils.ex @@ -0,0 +1,11 @@ +defmodule Mix.Tasks.Utils do + @moduledoc false + + def apply_function_if_key_exists(opts, key, func, args) do + if opts[key] do + apply(func, args) + else + :ok + end + end +end diff --git a/lib/mix/tasks/validate.ex b/lib/mix/tasks/validate.ex new file mode 100644 index 000000000..e76f31b15 --- /dev/null +++ b/lib/mix/tasks/validate.ex @@ -0,0 +1,51 @@ +defmodule Mix.Tasks.Archethic.Proposal.Validator do + @shortdoc "Run regression utilities to benchmark and validate nodes containing code proposal" + + @moduledoc """ + The Archethic Code Proposal Validator mix task wrapper + + ## Command line options + + * `--help` - show this help + * `--phase=1` - launch phase 1 + * `--phase=2` - launch phase 2 + + ## Example + + ```sh + mix archethic.validate localhost + ``` + + """ + use Mix.Task + + alias Archethic.Governance.Code.Proposal.Validator + alias Mix.Tasks.Utils + + @impl Mix.Task + @spec run([binary]) :: any + def run(args) do + Application.ensure_all_started(:telemetry) + + case OptionParser.parse!(args, + strict: [ + help: :boolean, + phase: :integer + ] + ) do + {_, []} -> + Mix.shell().cmd("mix help #{Mix.Task.task_name(__MODULE__)}") + + {parsed, nodes} -> + if parsed[:help] do + Mix.shell().cmd("mix help #{Mix.Task.task_name(__MODULE__)}") + else + :ok = + Utils.apply_function_if_key_exists(parsed, :phase, &Validator.run/2, [ + nodes, + parsed[:phase] + ]) + end + end + end +end diff --git a/mix.exs b/mix.exs index aae295bdf..8960794b2 100644 --- a/mix.exs +++ b/mix.exs @@ -10,7 +10,7 @@ defmodule Archethic.MixProject do deps_path: "deps", lockfile: "mix.lock", aliases: aliases(), - elixir: "~> 1.11", + elixir: "~> 1.14", start_permanent: Mix.env() == :prod, deps: deps(), compilers: [:elixir_make] ++ Mix.compilers(), @@ -74,7 +74,7 @@ defmodule Archethic.MixProject do {:benchee_html, "~> 1.0", only: :dev}, {:ex_doc, "~> 0.29", only: :dev, runtime: false}, {:git_hooks, "~> 0.7", runtime: false}, - {:credo, "~> 1.6", only: [:dev, :test], runtime: false}, + {:credo, "~> 1.6", runtime: false}, {:elixir_make, "~> 0.6", runtime: false}, {:dialyxir, "~> 1.2", runtime: false}, {:logger_file_backend, "~> 0.0.13", only: :dev}, @@ -82,7 +82,7 @@ defmodule Archethic.MixProject do {:dart_sass, "~> 0.5", runtime: Mix.env() == :dev}, # Security - {:sobelow, "~> 0.11", only: [:test, :dev], runtime: false}, + {:sobelow, "~> 0.11", runtime: false}, # Test {:mox, "~> 1.0", only: [:test]}, diff --git a/mix.lock b/mix.lock index afc743fe4..2d07d4540 100644 --- a/mix.lock +++ b/mix.lock @@ -22,7 +22,7 @@ "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, "dialyxir": {:hex, :dialyxir, "1.2.0", "58344b3e87c2e7095304c81a9ae65cb68b613e28340690dfe1a5597fd08dec37", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "61072136427a851674cab81762be4dbeae7679f85b1272b6d25c3a839aff8463"}, "digital_token": {:hex, :digital_token, "0.4.0", "2ad6894d4a40be8b2890aad286ecd5745fa473fa5699d80361a8c94428edcd1f", [:mix], [{:cldr_utils, "~> 2.17", [hex: :cldr_utils, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "a178edf61d1fee5bb3c34e14b0f4ee21809ee87cade8738f87337e59e5e66e26"}, - "distillery": {:git, "https://github.com/archethic-foundation/distillery.git", "a85f23713c2914096293b25f256e578d40514189", []}, + "distillery": {:git, "https://github.com/archethic-foundation/distillery.git", "eb279d7e01f4e7cb2f77d96dfa2b8dd7c760e661", []}, "earmark": {:hex, :earmark, "1.4.35", "e067aab15367c6e43230d6a7409c5230403a48b56f7dcefb3abdad75b498289e", [:mix], [{:earmark_parser, "~> 1.4.30", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "ab869cad78ebe64a62d45ee31addc52fb703c5d595868c9aa11ca38766ff9756"}, "earmark_parser": {:hex, :earmark_parser, "1.4.30", "0b938aa5b9bafd455056440cdaa2a79197ca5e693830b4a982beada840513c5f", [:mix], [], "hexpm", "3b5385c2d36b0473d0b206927b841343d25adb14f95f0110062506b300cd5a1b"}, "easy_ssl": {:hex, :easy_ssl, "1.3.0", "472256942d9dd37652a558a789a8d1cccc27e7f46352e32667d1ca46bb9e22e5", [:mix], [], "hexpm", "ce8fcb7661442713a94853282b56cee0b90c52b983a83aa6af24686d301808e1"}, diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 000000000..3893c248b --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "archethic-node", + "lockfileVersion": 2, + "requires": true, + "packages": {} +} diff --git a/rel/commands/validate b/rel/commands/validate new file mode 100755 index 000000000..f367504d1 --- /dev/null +++ b/rel/commands/validate @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +release_ctl eval --mfa "Mix.Tasks.Archethic.Proposal.Validator.run/1" --argv -- "$@" diff --git a/rel/config.exs b/rel/config.exs index 8aef01704..3725bb148 100644 --- a/rel/config.exs +++ b/rel/config.exs @@ -32,11 +32,12 @@ environment Mix.env() do ] set overlays: [ - {:copy, "config/#{Mix.env()}.exs", "releases/<%= release_version %>/runtime_config.exs"} + {:copy, "config/#{Mix.env()}.exs", "releases/<%= release_version %>/runtime_config.exs"}, ] set commands: [ - regression_test: "rel/commands/regression_test" + regression_test: "rel/commands/regression_test", + validate: "rel/commands/validate" ] plugin Distillery.Releases.Plugin.CookieLoader diff --git a/scripts/governance/proposal_ci_job.sh b/scripts/governance/proposal_ci_job.sh index 99795d207..abf948267 100755 --- a/scripts/governance/proposal_ci_job.sh +++ b/scripts/governance/proposal_ci_job.sh @@ -1,14 +1,14 @@ #!/bin/bash +# redirect stdout/stderr to a file +exec >ci_logfile.txt 2>&1 + set -e PROPOSAL_ADDRESS=$1 PROPOSAL_DESCRIPTION=$2 PROPOSAL_FILENAME=./proposal.diff -echo "=== Test proposal ${PROPOSAL_ADDRESS}" -tee ${PROPOSAL_FILENAME} - echo "=== Create branch ${PROPOSAL_ADDRESS}" git checkout -b "prop_${PROPOSAL_ADDRESS}" @@ -18,11 +18,39 @@ git apply ${PROPOSAL_FILENAME} echo "=== git add files" git add --all -echo "=== git commit " +echo "=== git commit" git commit -m "${PROPOSAL_DESCRIPTION}" -echo "=== Run CI" -mix git_hooks.run pre_push +echo "=== Run CI START ===" + +echo "=== Run CI -- Part 1 -- clean " +mix clean + +echo "=== Run CI -- Part 2 -- format" +mix format --check-formatted + +echo "=== Run CI -- 3 -- compile" +mix compile --warnings-as-errors + +echo "=== Run CI -- 4 -- credo" +mix credo + +echo "=== Run CI -- 5 -- sobelow" +mix sobelow + +echo "=== Run CI -- 6 -- knigge" +mix knigge.verify + +echo "=== Run CI -- 7 -- test" +MIX_ENV=test mix test --trace + +echo "=== Run CI -- 8 -- dialyzer" +mix dialyzer + +echo "=== Run CI -- 9 -- updates" +mix check.updates + +echo "=== Run CI DONE ===" -echo "=== Create upgrade" -mix distillery.release --upgrade +echo "=== Create upgrade release" +MIX_ENV=prod mix distillery.release --upgrade diff --git a/src/c/nat/miniupnp b/src/c/nat/miniupnp new file mode 160000 index 000000000..49991e00f --- /dev/null +++ b/src/c/nat/miniupnp @@ -0,0 +1 @@ +Subproject commit 49991e00f6ba53f7fa836c3afec68e2939e80a31