Merge pull request #5063 from Joystream/nara

Upgrade to Nara

.env

@@ -1,5 +1,4 @@
 COMPOSE_PROJECT_NAME=joystream
-PROJECT_NAME=query_node
 
 # We use a single postgres service for both the query node indexer and processor.
 # The default `DB_*` environment variables point to query node processor's database.

.github/workflows/create-release.yml

@@ -65,18 +65,18 @@ jobs:
 
           docker run -d --entrypoint tail --name temp-container-joystream-node $IMAGE-amd64 -f /dev/null
 
-          RESULT=$(docker exec temp-container-joystream-node b2sum -l 256 runtime.compact.wasm | awk '{print $1}')
-          VERSION_AND_COMMIT=$(docker exec temp-container-joystream-node /joystream/node --version | awk '{print $2}' | cut -d- -f -2)
+          RESULT=`docker exec temp-container-joystream-node b2sum -l 256 runtime.compact.compressed.wasm | awk '{print $1}'`
+          VERSION_AND_COMMIT=`docker exec temp-container-joystream-node /joystream/node --version | awk '{print $2}' | cut -d- -f -2`
           echo "::set-output name=blob_hash::${RESULT}"
           echo "::set-output name=version_and_commit::${VERSION_AND_COMMIT}"
 
-          docker cp temp-container-joystream-node:/joystream/runtime.compact.wasm ./joystream_runtime_${{ github.event.inputs.tag }}.wasm
+          docker cp temp-container-joystream-node:/joystream/runtime.compact.compressed.wasm ./joystream_runtime_${{ github.event.inputs.tag }}.wasm
           docker cp temp-container-joystream-node:/joystream/node ./joystream-node
           tar -czvf joystream-node-$VERSION_AND_COMMIT-x86_64-linux-gnu.tar.gz joystream-node
 
           docker rm --force temp-container-joystream-node
 
-          docker cp $(docker create --rm $IMAGE-arm64):/joystream/node ./joystream-node
+          docker cp `docker create --rm $IMAGE-arm64`:/joystream/node ./joystream-node
           tar -czvf joystream-node-$VERSION_AND_COMMIT-arm64-linux-gnu.tar.gz joystream-node
 
       - name: Retrieve saved MacOS binary

.github/workflows/deploy-node-network.yml

@@ -94,9 +94,9 @@ jobs:
         run: |
           VAL1="${{ steps.deploy_stack.outputs.Val1PublicIp }}"
           VAL2="${{ steps.deploy_stack.outputs.Val2PublicIp }}"
-          VAL3="${{ steps.deploy_stack.outputs.Val3PublicIp }}"
+          BOOT1="${{ steps.deploy_stack.outputs.Val3PublicIp }}"
           echo -e "[validators]\n$VAL1\n$VAL2\n\n" >> inventory
-          echo -e "[boot]\n$VAL3\n\n" >> inventory
+          echo -e "[boot]\n$BOOT1\n\n" >> inventory
           echo -e "[build]\n${{ steps.deploy_stack.outputs.BuildPublicIp }}\n\n" >> inventory
           echo -e "[rpc]\n${{ steps.deploy_stack.outputs.RPCPublicIp }}\n" >> inventory
           cat inventory

.github/workflows/deploy-playground.yml

@@ -12,7 +12,7 @@ on:
         required: false
         default: 'master'
       runtimeProfile:
-        description: 'STAGING | PLAYGROUND | TESTING - leave balnk for production'
+        description: 'PLAYGROUND | TESTING'
         default: 'PLAYGROUND'
         required: false
       sshPubKey:

.github/workflows/joystream-apps-docker.yml

@@ -25,9 +25,9 @@ jobs:
         id: extract_versions
         shell: bash
         run: |
-          echo "colossus_version=$(cat storage-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT
-          echo "argus_version=$(cat distributor-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT
-          echo "qn_version=$(cat query-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT
+          echo "colossus_version=`cat storage-node/package.json | jq -r '.version'`" >> $GITHUB_OUTPUT
+          echo "argus_version=`cat distributor-node/package.json | jq -r '.version'`" >> $GITHUB_OUTPUT
+          echo "qn_version=`cat query-node/package.json | jq -r '.version'`" >> $GITHUB_OUTPUT
 
       - name: Make some space
         shell: bash
@@ -48,7 +48,10 @@ jobs:
 
       - name: Build joystream-node
         if: github.event_name == 'pull_request'
-        run: RUNTIME_PROFILE=TESTING ./build-node-docker.sh
+        run: |
+          RUNTIME_PROFILE=TESTING ./build-node-docker.sh
+          # reclaim some disk space
+          docker system prune --force
 
       # docker/build-push-action doc:
       # Be careful because any file mutation in the steps that precede the
@@ -65,6 +68,8 @@ jobs:
           push: false
           load: true
           tags: joystream/storage-node:latest
+      - name: reclaim disk space
+        run: docker system prune --force
       - name: Build distributor-node
         uses: docker/build-push-action@v3
         with:
@@ -74,6 +79,8 @@ jobs:
           push: false
           load: true
           tags: joystream/distributor-node:latest
+      - name: reclaim disk space
+        run: docker system prune --force
       - name: Build query-node
         uses: docker/build-push-action@v3
         with:
@@ -87,6 +94,7 @@ jobs:
       - name: Test with latest images
         if: github.event_name == 'pull_request'
         run: |
+          docker system prune --force
           yarn build
           cp docker-compose-no-bind-volumes.yml docker-compose.yml
           ./tests/network-tests/run-tests.sh content-directory

.github/workflows/joystream-node-docker-dev.yml

@@ -15,10 +15,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        runtime_profile: ['STAGING', 'PLAYGROUND', 'TESTING']
+        runtime_profile: ['FAST-PROD', 'PLAYGROUND', 'TESTING']
         include:
-          - runtime_profile: 'STAGING'
-            cargo_features: 'staging-runtime'
+          - runtime_profile: 'FAST-PROD'
+            cargo_features: 'warp-time'
           - runtime_profile: 'PLAYGROUND'
             cargo_features: 'playground-runtime'
           - runtime_profile: 'TESTING'
@@ -27,6 +27,13 @@ jobs:
       RUNTIME_PROFILE: ${{ matrix.runtime_profile }}
 
     steps:
+      - name: Make some space
+        shell: bash
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
       - name: Checkout sources
         uses: actions/checkout@v3
       - id: compute_shasum
@@ -47,15 +54,37 @@ jobs:
           export IMAGE_EXISTS=$(docker manifest inspect ${{ env.REPOSITORY }}:${{ steps.compute_shasum.outputs.shasum }} > /dev/null ; echo $?)
           echo "::set-output name=image_exists::${IMAGE_EXISTS}"
 
+      # Emulation only needed if cross compiling
+      # - name: Set up QEMU
+      #   uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        # uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1.7
+        uses: docker/setup-buildx-action@v2
+        id: builder1
+        if: ${{ steps.compute_image_exists.outputs.image_exists == 1 }}
+
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-dev
+        if: ${{ steps.compute_image_exists.outputs.image_exists == 1 }}
+
       - name: Build and push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: .
           file: joystream-node.Dockerfile
+          # arm64 cross compiling takes longer than 6h, so we only build for amd64
+          # platforms: linux/amd64,linux/arm64
           platforms: linux/amd64
           build-args: |
             CARGO_FEATURES=${{ matrix.cargo_features }}
             CODE_SHASUM=${{ steps.compute_shasum.outputs.shasum }}
           push: true
           tags: ${{ env.REPOSITORY }}:${{ steps.compute_shasum.outputs.shasum }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+          builder: ${{ steps.builder1.outputs.name }}
         if: ${{ steps.compute_image_exists.outputs.image_exists == 1 }}

.github/workflows/joystream-node.yml

@@ -36,16 +36,15 @@ jobs:
 
       - name: Build
         env:
-          WASM_BUILD_TOOLCHAIN: nightly-2022-05-11 
+          WASM_BUILD_TOOLCHAIN: nightly-2022-11-15
         run: |
-          cargo +nightly-2022-05-11 clippy --release --features runtime-benchmarks &&
-          cargo +nightly-2022-05-11 test --release --features runtime-benchmarks &&
-          cargo +nightly-2022-05-11 build --release --features runtime-benchmarks
+          ./scripts/cargo-checks-with-benchmarking.sh &&
+          ./scripts/cargo-build-with-benchmarking.sh
         if: env.GIT_DIFF
 
       - name: Test Generate Weights
         run: |
-          ./scripts/generate-weights.sh 1 1 && git diff
+          ./scripts/generate-weights.sh 2 1 && git diff
         if: env.GIT_DIFF
 
   # Build and test joystream-node with all runtime profiles
@@ -96,3 +95,47 @@ jobs:
         run: |
           yarn cargo-checks && yarn cargo-build
         if: env.GIT_DIFF
+      - name: Fast Production Runtime
+        env:
+          RUNTIME_PROFILE: "FAST-PROD"
+        run: |
+          yarn cargo-checks && yarn cargo-build
+        if: env.GIT_DIFF
+
+  # Test runtime upgrade with "try-runtime" feature
+  runtime_upgrade:
+    name: Runtime Upgrade
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.x'
+      - uses: technote-space/get-diff-action@v3
+        with:
+          PREFIX_FILTER: |
+            bin
+            runtime
+            runtime-modules
+          SUFFIX_FILTER: |
+            .rs
+          FILES: |
+            Cargo.lock
+            Cargo.toml
+      - name: Toolchains
+        run: |
+          ./setup.sh
+        if: env.GIT_DIFF
+      - name: OnRuntimeUpgrade (try-runtime)
+        run: |
+          cargo +nightly-2022-11-15 build --release --features try-runtime
+          RUST_LOG=info,runtime=debug ./target/release/joystream-node try-runtime \
+            --runtime ./target/release/wbuild/joystream-node-runtime/joystream_node_runtime.compact.compressed.wasm \
+            on-runtime-upgrade \
+            live \
+            --uri wss://rpc.joystream.org:9944/
+        if: env.GIT_DIFF
+      - name: run_migrations test
+        run: |
+          RUN_MIGRATION_TESTS=true RUST_LOG=info cargo +nightly-2022-11-15 test remote_tests::run_migrations --release --features try-runtime
+        if: env.GIT_DIFF

.github/workflows/run-network-tests.yml

@@ -47,6 +47,13 @@ jobs:
       use_artifact: ${{ steps.compute_shasum.outputs.shasum }}-joystream-node-docker-image.tar.gz
       runtime: ${{ steps.compute_shasum.outputs.shasum }}
     steps:
+      - name: Make some space
+        shell: bash
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
@@ -104,7 +111,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx
+          key: ${{ runner.os }}-buildx-tests
         if: steps.check_files.outputs.files_exists == 'false'
 
       - name: Build
@@ -179,7 +186,11 @@ jobs:
     name: Runtime Upgrade from production runtime
     needs: build_images
     runs-on: ubuntu-latest
-    if: github.ref != 'refs/heads/master'
+    # Disabling until we find a workaround
+    # as it is no longer practical to start the node
+    # while importing huge state from production network in genesis block.
+    # if: github.ref != 'refs/heads/master'
+    if: false
     steps:
       # Checkout master branch
       - name: check out master repo

.pipelines/deploy-node-network-inputs.json

@@ -5,7 +5,7 @@
   },
   "branchName": {
     "description": "Branch to deploy",
-    "value": "carthage"
+    "value": "master"
   },
   "instanceType": {
     "description": "AWS EC2 instance type for Validators and Rpc (t2.micro, t2.large, t2.xlarge)",
@@ -28,15 +28,15 @@
     "value": ""
   },
   "deploymentType": {
-    "description": "Chain deployment type (live, dev etc.)",
-    "value": "staging"
+    "description": "Chain deployment type (testnet, mainnet)",
+    "value": "testnet"
   },
   "encryptionKey": {
     "description": "Password to encrypt the artifacts",
-    "value": "staging"
+    "value": "testnet"
   },
   "runtimeProfile": {
-    "description": "STAGING | PLAYGROUND | TESTING - leave empty for production",
+    "description": "STAGING | PLAYGROUND | TESTING | FAST-PROD (leave empty for production)",
     "value": "PLAYGROUND"
   },
   "endowAccounts": {