Feature/simnode workflow (#461)

* Update Makefile * Update release * Update * Updating release * Added runtime check * Consolidate all branches into main * Initalize cache * Initalize cache * Initalize cache * Initalize cache * Initalize cache * Initalize cache * Initalize cache * Initalize cache * Initalize cache * Initalize cache * Initalize cache * Initalize cache * Update build parameters * Update build parameters * Add simnode workflow * Add simonde to workflow file * Add simonde to workflow file * Update ci * Update ci * Update cargo path * Update cargo path * Update cargo path * Update cargo path * Update cargo path
ComposableFi · Jan 10, 2022 · 526263a · 526263a
1 parent a532ead
commit 526263a
Show file tree

Hide file tree

Showing 9 changed files with 88 additions and 118 deletions.
diff --git a/.github/workflows/Picasso-release.yml b/.github/workflows/Picasso-release.yml
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
@@ -21,23 +21,18 @@ jobs:
       - x64
       - sre
     steps:
-      - uses: xanantis/docker-file-ownership-fix@v1
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0     
-      - name: Setup toolchain
-        run: |
-         curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-         echo "$HOME/.cargo/bin:$PATH" >> $GITHUB_PATH
       - name: Check
         run: |
-          SKIP_WASM_BUILD=1 cargo check --all --benches
+          SKIP_WASM_BUILD=1 /home/runner/.cargo/bin/cargo check --all --benches
       - name: Benchmark
         run: |
-          SKIP_WASM_BUILD=1 cargo check --features=runtime-benchmarks --workspace --release --exclude integration-tests
+          SKIP_WASM_BUILD=1 /home/runner/.cargo/bin/cargo check --features=runtime-benchmarks --workspace --release --exclude integration-tests
       - name: Picasso
         run: |
-          cargo build --release -p picasso-runtime
+          /home/runner/.cargo/bin/cargo build --release -p picasso-runtime
   linters:
     name: Linters
     runs-on:
@@ -47,18 +42,13 @@ jobs:
       - sre
     steps:
       - uses: actions/checkout@v2
-      - name: Setup toolchain
-        run: |
-         sudo apt-get install libfreetype6-dev -y
-         curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-         echo "$HOME/.cargo/bin:$PATH" >> $GITHUB_PATH
       - run: |
-          rustup show
-          rustup override unset
-          rustup show
+          /home/runner/.cargo/bin/rustup show
+          /home/runner/.cargo/bin/rustup override unset
+          /home/runner/.cargo/bin/rustup show
       - name: Cargo fmt
         run: |
-          cargo fmt -- --check
+          /home/runner/.cargo/bin/cargo fmt -- --check
       - name: Hadolint
         env:
           HADOLINT_VERSION: v2.8.0
@@ -75,7 +65,7 @@ jobs:
           exit $total_exit_code
       - name: Cargo clippy
         run: |
-          cargo clippy -- -D warnings
+          /home/runner/.cargo/bin/cargo clippy -- -D warnings
       - name: Cargo udeps
         env:
           UDEPS_VERSION: v0.1.24
@@ -96,19 +86,20 @@ jobs:
       - sre
     steps:
       - uses: actions/checkout@v2
-      - name: Install toolchain for cargo-llvm-cov
-        run: |
-         curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-         echo "$HOME/.cargo/bin:$PATH" >> $GITHUB_PATH
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+
+      - uses: actions/checkout@v2
       - run: |
-          rustup component add llvm-tools-preview
+          /home/runner/.cargo/bin/rustup component add llvm-tools-preview
       - name: Install cargo-llvm-cov
         run: curl -LsSf https://github.com/taiki-e/cargo-llvm-cov/releases/latest/download/cargo-llvm-cov-x86_64-unknown-linux-gnu.tar.gz | tar xzf - -C ~/.cargo/bin
       - name: Run Test (with coverage)
         run: |
-          SKIP_WASM_BUILD=1 cargo llvm-cov --workspace --lcov --output-path lcov.info
-          SKIP_WASM_BUILD=1 cargo clean -p picasso-runtime
-          SKIP_WASM_BUILD=1 cargo clean -p composable
+          SKIP_WASM_BUILD=1 /home/runner/.cargo/bin/cargo llvm-cov --workspace --lcov --output-path lcov.info
+          SKIP_WASM_BUILD=1 /home/runner/.cargo/bin/cargo clean -p picasso-runtime
+          SKIP_WASM_BUILD=1 /home/runner/.cargo/bin/cargo clean -p composable
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1
         with:

diff --git a/.github/workflows/client-release.yml b/.github/workflows/client-release.yml
@@ -1,14 +1,17 @@
-name: "Dali-chachacha Community Release"
+name: "Community Release (Dali Chacha)"
 
 on:
   push: 
     tags: 
       - '*'
     branches:
-      - update-event-triggers
+      - feature/add-cache-build
 
 env:
   DOCKER_USER_OPTION: '$UID:$GID'
+  SCCACHE_GCS_BUCKET: 'composable-build-artefacts'
+  RUSTC_WRAPPER: "/home/runner/.cargo/bin/sccache"
+  SCCACHE_GCS_RW_MODE: "READ_WRITE"
 
 jobs:
   build-and-publish:
@@ -17,6 +20,8 @@ jobs:
         - linux
         - x64
         - sre
+    strategy: 
+      fail-fast: true
     steps:
     - uses: xanantis/docker-file-ownership-fix@v1
     - uses: actions/checkout@v2
@@ -27,9 +32,20 @@ jobs:
       run: |
         echo "RELEASE_VERSION=$(git tag --sort=committerdate | grep -E '^v[0-9]' | tail -1 )" >> $GITHUB_ENV
         make version
-    - name: Build and Push Artifacts to gcloud
+
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+
+    - name: 🔨 Build
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        args: --release --bins
+
+    - name: Archive build targets
       run: |
-        /home/runner/.cargo/bin/cargo build --release --bins
+        tar -czvf composable-${{ env.RELEASE_VERSION }}.tar.gz target/release/
         tar -czvf composable-${{ env.RELEASE_VERSION }}.tar.gz target/release/composable
         tar -czvf picasso_runtime.compact.wasm-${RELEASE_VERSION}.tar.gz target/release/wbuild/picasso-runtime/picasso_runtime.compact.wasm
         tar -czvf parachain-utils-${RELEASE_VERSION}.tar.gz target/release/parachain-utils
@@ -43,6 +59,7 @@ jobs:
 
     - name: Build Docker image
       run: make containerize-release 
+
     - name:  Docker push
       run: make push-release
 

diff --git a/.github/workflows/picashot.yml b/.github/workflows/picashot.yml
@@ -0,0 +1,18 @@
+name: Check Runtime
+on:
+  schedule:
+    - cron:  '0 0 * * *'
+jobs:
+  picashot:
+    runs-on: 
+      - self-hosted
+      - linux
+      - X64
+      - ansible-runner
+
+    steps:
+    - name: Run ansible playbook
+      working-directory: "./.maintain/playbooks"
+      run: |
+        ansible-playbook -l _mainnet_collator_yes picashot.yml -i picasso-gcp.yaml  --user root -e 'ansible_python_interpreter=/usr/bin/python3'
+
diff --git a/.github/workflows/runtime-check.yml b/.github/workflows/runtime-check.yml
@@ -2,10 +2,12 @@ name: Check Runtime
 on:
   push:
     branches:
-      - release/*
+      - release-*
+      - main
   pull_request:
     branches:
-      - release/*
+      - release-*
+      - main
 jobs:
   check-runtime:
     runs-on: ubuntu-latest

diff --git a/.maintain/check_runtime.sh b/.maintain/check_runtime.sh
@@ -16,40 +16,13 @@ VERSIONS_FILE="runtime/picasso/src/lib.rs"
 boldprint () { printf "|\n| \033[1m%s\033[0m\n|\n" "${@}"; }
 boldcat () { printf "|\n"; while read -r l; do printf "| \033[1m%s\033[0m\n" "${l}"; done; printf "|\n" ; }
 
-<<<<<<< HEAD
-github_label () {
-	echo
-	echo "# run github-api job for labeling it ${1}"
-	curl -sS -X POST \
-		-F "token=${CI_JOB_TOKEN}" \
-		-F "ref=main" \
-		-F "variables[LABEL]=${1}" \
-		-F "variables[PRNO]=${CI_COMMIT_REF_NAME}" \
-		"${GITLAB_API}/projects/${GITHUB_API_PROJECT}/trigger/pipeline"
-}
-
-
-boldprint "latest 10 commits of ${CI_COMMIT_REF_NAME}"
-git log --graph --oneline --decorate=short -n 10
-
-boldprint "make sure the main branch and dali-chachacha tag are available in shallow clones"
-git fetch --depth="${GIT_DEPTH:-100}" origin main
-git fetch --depth="${GIT_DEPTH:-100}" origin dali-chachacha
-git tag -f dali-chachacha FETCH_HEAD
-git log -n1 dali-chachacha
-
-
-boldprint "check if the wasm sources changed"
-if ! has_runtime_changes origin/master "${CI_COMMIT_SHA}"
-=======
 
 boldprint "latest 10 commits of ${GITHUB_REF_NAME}"
 git log --graph --oneline --decorate=short -n 10
 
 
 boldprint "check if the wasm sources changed"
 if ! has_runtime_changes origin/main "${GITHUB_REF_NAME}"
->>>>>>> bbc78797a884a247a0eb17742f1eda5975252d60
 then
 	boldcat <<-EOT
 
@@ -66,15 +39,9 @@ fi
 # consensus-critical logic that has changed. the runtime wasm blobs must be
 # rebuilt.
 
-<<<<<<< HEAD
-add_spec_version="$(git diff tags/release ${CI_COMMIT_SHA} -- "${VERSIONS_FILE}" \
-	| sed -n -r "s/^\+[[:space:]]+spec_version: +([0-9]+),$/\1/p")"
-sub_spec_version="$(git diff tags/release ${CI_COMMIT_SHA} -- "${VERSIONS_FILE}" \
-=======
 add_spec_version="$(git diff tags/release ${GITHUB_SHA} -- "${VERSIONS_FILE}" \
 	| sed -n -r "s/^\+[[:space:]]+spec_version: +([0-9]+),$/\1/p")"
 sub_spec_version="$(git diff tags/release ${GITHUB_SHA} -- "${VERSIONS_FILE}" \
->>>>>>> bbc78797a884a247a0eb17742f1eda5975252d60
 	| sed -n -r "s/^\-[[:space:]]+spec_version: +([0-9]+),$/\1/p")"
 
 
@@ -95,15 +62,9 @@ else
 	# check for impl_version updates: if only the impl versions changed, we assume
 	# there is no consensus-critical logic that has changed.
 
-<<<<<<< HEAD
-	add_impl_version="$(git diff tags/release ${CI_COMMIT_SHA} -- "${VERSIONS_FILE}" \
-		| sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p')"
-	sub_impl_version="$(git diff tags/release ${CI_COMMIT_SHA} -- "${VERSIONS_FILE}" \
-=======
 	add_impl_version="$(git diff tags/release ${GITHUB_SHA} -- "${VERSIONS_FILE}" \
 		| sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p')"
 	sub_impl_version="$(git diff tags/release ${GITHUB_SHA} -- "${VERSIONS_FILE}" \
->>>>>>> bbc78797a884a247a0eb17742f1eda5975252d60
 		| sed -n -r 's/^\-[[:space:]]+impl_version: +([0-9]+),$/\1/p')"
 
 

diff --git a/.maintain/files/backup.sh b/.maintain/files/backup.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+DATE=$(date +'%m-%d-%Y')
+FILENAME=composable-$DATE.zip
+BACKUP_DIR="/var/lib/composable-data/chains"
+GS_BUCKET="composable-picasso-data-sync"
+zip -r $FILENAME $BACKUP_DIR
+gsutil mv $FILENAME gs://$GS_BUCKET/
diff --git a/.maintain/playbooks/picashot.yml b/.maintain/playbooks/picashot.yml
@@ -0,0 +1,20 @@
+---
+- name: Create Backup for Picasso nodes
+  hosts: all
+  become: true
+  tasks:
+    - name: Run Backup Script
+      become: true
+      shell: |
+        #!/bin/bash
+        DATE=$(date +'%m-%d-%Y')
+        HOST=$(echo $HOSTNAME)
+        FILENAME=$HOST-$DATE.zip
+        BACKUP_DIR="/var/lib/composable-data/chains"
+        GS_BUCKET="composable-picasso-data-sync"
+        zip -r $FILENAME $BACKUP_DIR
+        gsutil mv $FILENAME gs://$GS_BUCKET/
+      args:
+        executable: /bin/bash
+      register: stdout
+
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -1,6 +1,6 @@
 FROM composablefi/rust:latest
 
-ARG VERSION=v1.0.9
+ARG VERSION
 
 USER root
 
@@ -24,8 +24,6 @@ RUN curl "https://storage.googleapis.com/composable-binaries/community-releases/
 
 RUN chmod +x target/release/composable && mv target/release/composable /usr/local/bin
 
-RUN composable --version
-
 RUN useradd -m -u 1001 -U -s /bin/sh -d /composable composable && \
     mkdir -p /composable/.local/share/local_testnet && \
     mkdir /data && \