Improve the code proposal workflow for continuous deployment (#955)

* fix cd pipeline for code proposals * fixing ci pipeline * fixed crypto root ca certificate value in test * fix logs redirection output * reduce docker images size * fix cd pipeline for code proposals * fix governance validation workflow * Use only 3 nodes for the CD to reduce memory usage * Fix containers names * bigger timeout for the upgrade part * remove tons of warning and debugs * log the error * bind the log file to the host /tmp folder * Leverage CI on code approval to trigger CD on governance pipeline * Remove CI container before to start --------- Co-authored-by: Samuel Manzanera <samuelmanzanera@protonmail.com> Co-authored-by: Bastien CHAMAGNE <bastien@chamagne.fr> Co-authored-by: Samuel Manzanera <samuelmanzanera@users.noreply.github.com>
archethic-foundation · Jun 5, 2023 · af0c186 · af0c186
1 parent 61da159
commit af0c186
Show file tree

Hide file tree

Showing 28 changed files with 537 additions and 314 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -2,72 +2,71 @@ name: Elixir CI
 
 on:
   push:
-    branches: [ master, develop, testnet ]
+    branches: [master, develop, testnet]
   pull_request:
-    branches: [ master, develop, testnet ]
+    branches: [master, develop, testnet]
 
 env:
   MIX_ENV: test
 
 jobs:
   build:
-
     name: Build and test
     runs-on: ubuntu-20.04
-    
+
     steps:
-    - name: Install OS Packages
-      uses: mstksg/get-package@2a4b48d55d72d43ca89ae58ec9ca1397d34a1c35
-      with:
-        apt-get: libgmp-dev libssl-dev libtinfo-dev libsystemd-dev zlib1g-dev libsodium-dev autoconf-archive libcmocka0 libcmocka-dev procps iproute2 build-essential git pkg-config gcc libtool automake libssl-dev uthash-dev autoconf doxygen libjson-c-dev libini-config-dev libcurl4-openssl-dev libltdl-dev libtss2-dev tss2
-    - uses: actions/checkout@v2
-    - name: Set up Elixir
-      uses: erlef/setup-beam@988e02bfe678367a02564f65ca2e37726dc0268f
-      id: beam
-      with:
-        elixir-version: '1.14.1' # Define the elixir version [required]
-        otp-version: '25.1' # Define the OTP version [required]
-    - name: Restore dependencies cache
-      uses: actions/cache@v3
-      id: mix-cache
-      with:
-        path: deps
-        key: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-mix-${{ hashFiles('**/mix.lock') }}
-        restore-keys: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-mix-
-    - name: Make Clean
-      run: mix clean
-    - name: Install dependencies
-      if: steps.mix-cache.outputs.cache-hit != 'true'
-      run: mix deps.get
-    - name: Check dependency updates
-      run: mix hex.outdated --within-requirements 1>/dev/null || echo 'Updates available!'
-    - name: Set Formatting
-      run: mix format --check-formatted
-    - name: Restore build cache
-      uses: actions/cache@v3
-      with:
-        path: _build
-        key: ${{ runner.os }}-build-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('**/mix.lock') }}
-        restore-keys: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-build-
-    - name: Compile the codebase
-      run: mix compile --warnings-as-errors
-    - name: Set credo
-      run: mix credo 
-    - name: Run Sobelow
-      run: mix sobelow 
-    - name: Run tests
-      run: mix test --trace
-    - name: Retrieve PLT Cache
-      uses: actions/cache@v3
-      id: plt-cache
-      with:
-        path: priv/plts
-        key: ${{ runner.os }}-plts-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('**/mix.lock') }}
-        restore-keys: ${{ runner.os }}-plts-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-
-    - name: Create PLTs
-      if: steps.plt-cache.outputs.cache-hit != 'true'
-      run: |
-        mkdir -p priv/plts
-        mix dialyzer --plt
-    - name: Run dialyzer
-      run: mix dialyzer --no-check --ignore-exit-status
+      - name: Install OS Packages
+        uses: mstksg/get-package@2a4b48d55d72d43ca89ae58ec9ca1397d34a1c35
+        with:
+          apt-get: libgmp-dev libssl-dev libtinfo-dev libsystemd-dev zlib1g-dev libsodium-dev autoconf-archive libcmocka0 libcmocka-dev procps iproute2 build-essential git pkg-config gcc libtool automake libssl-dev uthash-dev autoconf doxygen libjson-c-dev libini-config-dev libcurl4-openssl-dev libltdl-dev libtss2-dev tss2
+      - uses: actions/checkout@v2
+      - name: Set up Elixir
+        uses: erlef/setup-beam@988e02bfe678367a02564f65ca2e37726dc0268f
+        id: beam
+        with:
+          elixir-version: "1.14.1" # Define the elixir version [required]
+          otp-version: "25.1" # Define the OTP version [required]
+      - name: Restore dependencies cache
+        uses: actions/cache@v3
+        id: mix-cache
+        with:
+          path: deps
+          key: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-mix-${{ hashFiles('**/mix.lock') }}
+          restore-keys: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-mix-
+      - name: Make Clean
+        run: mix clean
+      - name: Install dependencies
+        if: steps.mix-cache.outputs.cache-hit != 'true'
+        run: mix deps.get
+      - name: Check dependency updates
+        run: mix hex.outdated --within-requirements 1>/dev/null || echo 'Updates available!'
+      - name: Set Formatting
+        run: mix format --check-formatted
+      - name: Restore build cache
+        uses: actions/cache@v3
+        with:
+          path: _build
+          key: ${{ runner.os }}-build-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('**/mix.lock') }}
+          restore-keys: ${{ runner.os }}-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-build-
+      - name: Compile the codebase
+        run: mix compile --warnings-as-errors
+      - name: Set credo
+        run: mix credo
+      - name: Run Sobelow
+        run: mix sobelow
+      - name: Run tests
+        run: mix test --trace
+      - name: Retrieve PLT Cache
+        uses: actions/cache@v3
+        id: plt-cache
+        with:
+          path: priv/plts
+          key: ${{ runner.os }}-plts-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('**/mix.lock') }}
+          restore-keys: ${{ runner.os }}-plts-elixir-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-
+      - name: Create PLTs
+        if: steps.plt-cache.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p priv/plts
+          mix dialyzer --plt
+      - name: Run dialyzer
+        run: mix dialyzer --no-check --ignore-exit-status
diff --git a/Dockerfile b/Dockerfile
@@ -1,7 +1,9 @@
 FROM elixir:1.14.1-alpine AS archethic-ci
 
-ARG skip_tests=0
+ARG with_tests=1
 ARG MIX_ENV=prod
+ARG USER_ID 
+ARG GROUP_ID
 
 # CI
 #  - compile
@@ -19,25 +21,22 @@ ARG MIX_ENV=prod
 #  - code
 #  - release
 
-# running TESTNET with release upgrade should ???
+ENV ARCHETHIC_NETWORK_TYPE=testnet
 
 RUN apk add --no-cache --update \
   build-base \
+  grep \
   bash \
   gcc \
   make \
   g++ \
-  libexecinfo-dev \
-  libexecinfo \
   git \
   npm \
-  python3 \
   wget \
   openssl \
   libsodium-dev \
-  gmp-dev \
-  miniupnpc
-
+  libexecinfo-dev \
+  gmp-dev 
 
 # Install hex and rebar
 RUN mix local.rebar --force \
@@ -61,23 +60,12 @@ RUN git config user.name aebot \
   && git config user.email aebot@archethic.net \
   && git remote add origin https://github.com/archethic-foundation/archethic-node
 
-# Install Dart Sass
-RUN npm install -g sass
-
-# build Sass -> CSS
-RUN cd assets && \
-  sass --no-source-map --style=compressed css/app.scss ../priv/static/css/app.css && cd -
-
 # build release
-RUN mix do assets.deploy, distillery.release
-
-# gen PLT
-RUN if [ $with_tests -eq 1 ]; then mix git_hooks.run pre_push ;fi
-
+RUN mix assets.deploy
+RUN MIX_ENV=${MIX_ENV} mix distillery.release
 # Install
 RUN mkdir -p /opt/app \
-  && cd /opt/app \
-  && tar zxf /opt/code/_build/${MIX_ENV}/rel/archethic_node/releases/*/archethic_node.tar.gz
+  && tar zxf /opt/code/_build/${MIX_ENV}/rel/archethic_node/releases/*/archethic_node.tar.gz -C  /opt/app
 CMD /opt/app/bin/archethic_node foreground
 
 ################################################################################
@@ -86,13 +74,19 @@ FROM archethic-ci as build
 
 FROM elixir:1.14.1-alpine
 
-RUN apk add --no-cache --update bash git openssl libsodium
+ARG USER_ID 
+ARG GROUP_ID
+
+RUN apk add --no-cache --update bash git openssl libsodium libexecinfo
 
 COPY --from=build /opt/app /opt/app
 COPY --from=build /opt/code/.git /opt/code/.git
 
 WORKDIR /opt/code
 RUN git reset --hard
 
+RUN rm -rf /opt/code/.git
+RUN rm -rf /opt/code/priv
+
 WORKDIR /opt/app
 CMD /opt/app/bin/archethic_node foreground
diff --git a/Makefile b/Makefile
@@ -15,7 +15,6 @@ ifeq ($(TPM_INSTALLED),0)
 	$(CC) src/c/crypto/tpm/keygen.c src/c/crypto/tpm/lib.c -o priv/c_dist/tpm_keygen -I src/c/crypto/tpm/lib.h $(TPMFLAGS)
 endif
 
-
 clean:
 	rm -f priv/c_dist/*
 	mix archethic.clean_db

diff --git a/assets/package-lock.json b/assets/package-lock.json
diff --git a/config/dev.exs b/config/dev.exs
@@ -167,13 +167,13 @@ config :archethic, ArchethicWeb.Endpoint,
 config :archethic, :throttle,
   by_ip_high: [
     period: 1000,
-    limit: 5_000
+    limit: System.get_env("ARCHETHIC_THROTTLE_IP_HIGH", "5000")
   ],
   by_ip_low: [
     period: 1000,
-    limit: 5_000
+    limit: System.get_env("ARCHETHIC_THROTTLE_IP_LOW", "5000")
   ],
   by_ip_and_path: [
     period: 1000,
-    limit: 5_000
+    limit: System.get_env("ARCHETHIC_THROTTLE_IP_AND_PATH", "5000")
   ]
diff --git a/config/prod.exs b/config/prod.exs
@@ -89,14 +89,21 @@ config :archethic, Archethic.BeaconChain.SummaryTimer,
 
 config :archethic, Archethic.Crypto,
   root_ca_public_keys: [
-    software: [
-      secp256r1:
-        System.get_env(
-          "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY",
-          "04F0FE701A03CE375A6E57ADBE0255808812036571C1424DB2779C77E8B4A9BA80A15B118E8E7465EE2E94094E59C4B3F7177E99063AF1B19BFCC4D7E1AC3F89DD"
-        )
-        |> Base.decode16!(case: :mixed)
-    ],
+    software:
+      case System.get_env("ARCHETHIC_NETWORK_TYPE") do
+        "testnet" ->
+          []
+
+        _ ->
+          [
+            secp256r1:
+              System.get_env(
+                "ARCHETHIC_CRYPTO_ROOT_CA_SOFTWARE_PUBKEY",
+                "04F0FE701A03CE375A6E57ADBE0255808812036571C1424DB2779C77E8B4A9BA80A15B118E8E7465EE2E94094E59C4B3F7177E99063AF1B19BFCC4D7E1AC3F89DD"
+              )
+              |> Base.decode16!(case: :mixed)
+          ]
+      end,
     tpm: [
       secp256r1:
         System.get_env(
@@ -273,13 +280,13 @@ config :archethic, ArchethicWeb.Endpoint,
 config :archethic, :throttle,
   by_ip_high: [
     period: 1000,
-    limit: 500
+    limit: System.get_env("ARCHETHIC_THROTTLE_IP_HIGH", "500")
   ],
   by_ip_low: [
     period: 1000,
-    limit: 20
+    limit: System.get_env("ARCHETHIC_THROTTLE_IP_LOW", "20")
   ],
   by_ip_and_path: [
     period: 1000,
-    limit: 20
+    limit: System.get_env("ARCHETHIC_THROTTLE_IP_AND_PATH", "20")
   ]
diff --git a/config/test.exs b/config/test.exs
@@ -179,13 +179,13 @@ config :archethic, ArchethicWeb.Endpoint,
 config :archethic, :throttle,
   by_ip_high: [
     period: 1000,
-    limit: 5_000
+    limit: System.get_env("ARCHETHIC_THROTTLE_IP_HIGH", "5000")
   ],
   by_ip_low: [
     period: 1000,
-    limit: 5_000
+    limit: System.get_env("ARCHETHIC_THROTTLE_IP_LOW", "5000")
   ],
   by_ip_and_path: [
     period: 1000,
-    limit: 5_000
+    limit: System.get_env("ARCHETHIC_THROTTLE_IP_AND_PATH", "5000")
   ]
diff --git a/lib/archethic/governance.ex b/lib/archethic/governance.ex
@@ -5,15 +5,21 @@ defmodule Archethic.Governance do
   """
 
   alias Archethic.Crypto
+  alias Archethic.Election
 
   alias __MODULE__.Code
   alias __MODULE__.Code.Proposal
   alias __MODULE__.Pools
 
+  alias Archethic.P2P
+
   alias Archethic.TransactionChain
   alias Archethic.TransactionChain.Transaction
   alias Archethic.TransactionChain.TransactionData
 
+  alias Archethic.TaskSupervisor
+  alias Archethic.Utils
+
   @proposal_tx_select_fields [
     :address,
     :timestamp,
@@ -105,11 +111,20 @@ defmodule Archethic.Governance do
           recipients: [prop_address]
         }
       }) do
-    if Code.testnet_deployment?(prop_address) do
-      {:ok, prop} = get_code_proposal(prop_address)
-      Code.deploy_proposal_testnet(prop)
+    storage_nodes =
+      Election.chain_storage_nodes(prop_address, P2P.authorized_and_available_nodes())
+
+    with true <- Utils.key_in_node_list?(storage_nodes, Crypto.first_node_public_key()),
+         {:ok, prop} <- get_code_proposal(prop_address),
+         true <- Code.enough_code_approval?(prop) do
+      Task.Supervisor.start_child(TaskSupervisor, fn ->
+        if Code.valid_integration?(prop) do
+          Code.deploy_proposal_testnet(prop)
+        end
+      end)
     else
-      :ok
+      _ ->
+        :ok
     end
   end