build bitacross docker image

litentry · Jul 9, 2024 · d100fcd · d100fcd
1 parent d7f5dc2
commit d100fcd
Show file tree

Hide file tree

Showing 3 changed files with 233 additions and 52 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -503,6 +503,148 @@ jobs:
         if: failure()
         uses: andymckay/cancel-action@0.5
 
+  bitacross-build:
+    runs-on: ubuntu-latest
+    needs:
+      - fmt
+      - set-condition
+      - sequentialise
+    steps:
+      - name: Free up disk space
+        if: startsWith(runner.name, 'GitHub Actions')
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
+          swap-storage: false
+          large-packages: false
+
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          # use the docker driver to access the local image
+          # we don't need external caches or multi platforms here
+          # see https://docs.docker.com/build/drivers/
+          driver: docker
+
+      - name: Cache worker-cache
+        if: needs.set-condition.outputs.rebuild_bitacross == 'true'
+        uses: actions/cache@v4
+        with:
+          path: |
+            worker-cache
+          key: worker-cache-${{ env.REF_VERSION }}-${{ hashFiles('bitacross-worker/**/Cargo.lock', 'bitacross-worker/**/Cargo.toml') }}
+          restore-keys: |
+            worker-cache-${{ env.REF_VERSION }}-
+            worker-cache-
+
+      - name: Create cache folder if not exist
+        if: needs.set-condition.outputs.rebuild_bitacross == 'true'
+        run: |
+          for i in 'git/db' 'registry/cache' 'registry/index' 'sccache'; do
+            [ ! -d "worker-cache/$i" ] && mkdir -p "worker-cache/$i" || true
+            echo "hello" > worker-cache/$i/nix
+          done
+          echo "::group::List worker-cache size"
+          du -sh worker-cache/*
+          echo "::endgroup::"
+          echo "::group::Show disk usage"
+          df -h .
+          echo "::endgroup::"
+      # cache mount in buildkit won't be exported as image layers, so it doesn't work well with GHA cache, see
+      # https://github.com/moby/buildkit/issues/1512
+      # https://www.reddit.com/r/rust/comments/126xeyx/exploring_the_problem_of_faster_cargo_docker/
+      # https://hackmd.io/@kobzol/S17NS71bh (a great summary)
+      #
+      # the `reproducible-containers/buildkit-cache-dance` seems to be unstable too in my test
+      # sometimes it works, sometimes it results in empty cache, root cause unclear
+      #
+      # thus the persistence of rust cache (w/ sccache) is maintained manually
+      #
+      # there's no cache-from/to as the docker image layer is too large and it takes too long to sync
+      # moreoever, we are not so eager to have layer caches, as the most critical operation(cargo build)
+      # is "atomic" and can't be broken into layers.
+      - name: Build local builder
+        if: needs.set-condition.outputs.rebuild_bitacross == 'true'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: bitacross-worker/build.Dockerfile
+          tags: local-builder:latest
+          target: builder
+          build-args: |
+            WORKER_MODE_ARG=offchain            
+            ADDITIONAL_FEATURES_ARG=
+
+      - name: Copy caches from the built image
+        if: needs.set-condition.outputs.rebuild_bitacross == 'true'
+        run: |
+          echo "::group::Show disk usage"
+          df -h .
+          echo "::endgroup::"
+          echo "::group::docker images"
+          docker images --all
+          echo "::endgroup::"
+          echo "::group::copy cache out"
+          for i in 'git/db' 'registry/cache' 'registry/index'; do
+            b="${i%/*}"
+            rm -rf worker-cache/$i
+            docker cp "$(docker create --rm local-builder:latest):/opt/rust/$i" worker-cache/$b
+          done
+          rm -rf worker-cache/sccache
+          docker cp "$(docker create --rm local-builder:latest):/opt/rust/sccache" worker-cache
+          du -sh worker-cache/*
+          echo "::endgroup::"
+          echo "::group::df -h ."
+          df -h .
+          echo "::endgroup::"
+
+      - name: Build worker
+        if: needs.set-condition.outputs.rebuild_bitacross == 'true'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: bitacross-worker/build.Dockerfile
+          tags: litentry/bitacross-worker:latest
+          target: deployed-worker
+
+      - name: Build cli
+        if: needs.set-condition.outputs.rebuild_bitacross == 'true'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: bitacross-worker/build.Dockerfile
+          tags: litentry/bitacross-cli:latest
+          target: deployed-client
+
+      - name: Pull and tag worker and cli image optionally
+        if: needs.set-condition.outputs.rebuild_bitacross == 'false'
+        run: |
+          docker pull litentry/bitacross-worker:latest
+          docker pull litentry/bitacross-cli:latest
+
+      - run: docker images --all
+
+      - name: Test enclave
+        if: needs.set-condition.outputs.rebuild_bitacross == 'true'
+        # cargo test is not supported in the enclave
+        # see https://github.com/apache/incubator-teaclave-sgx-sdk/issues/232
+        run: docker run litentry/bitacross-worker:latest test --all
+
+      - name: Save docker images
+        run: docker save litentry/bitacross-worker:latest litentry/bitacross-cli:latest | gzip > litentry-bitacross.tar.gz
+
+      - name: Upload docker images
+        uses: actions/upload-artifact@v4
+        with:
+          name: litentry-bitacross
+          path: litentry-bitacross.tar.gz
+          if-no-files-found: error
+      - name: Fail early
+        if: failure()
+        uses: andymckay/cancel-action@0.5
+
   parachain-ts-test:
     runs-on: ubuntu-latest
     needs:

diff --git a/bitacross-worker/cli/lit_ts_integration_test.sh b/bitacross-worker/cli/lit_ts_integration_test.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# Copyright 2020-2024 Trust Computing GmbH.
+
+set -euo pipefail
+
+while getopts ":p:A:u:W:V:C:" opt; do
+    case $opt in
+        p)
+            NPORT=$OPTARG
+            ;;
+        A)
+            WORKER1PORT=$OPTARG
+            ;;
+        u)
+            NODEURL=$OPTARG
+            ;;
+        W)
+            NODEHTTPURL=$OPTARG
+            ;;
+        V)
+            WORKER1URL=$OPTARG
+            ;;
+        C)
+            CLIENT_BIN=$OPTARG
+            ;;
+    esac
+done
+
+# Using default port if none given as arguments.
+NPORT=${NPORT:-9912}
+NODEURL=${NODEURL:-"ws://litentry-node"}
+NODEHTTPURL=${NODEHTTPURL:-"http://litentry-node"}
+WORKER1PORT=${WORKER1PORT:-2011}
+WORKER1URL=${WORKER1URL:-"ws://bitacross-worker-1"}
+
+CLIENT_BIN=${CLIENT_BIN:-"/usr/local/bin/bitacross-cli"}
+
+CLIENT="${CLIENT_BIN} -p ${NPORT} -P ${WORKER1PORT} -u ${NODEURL} -U ${WORKER1URL}"
+
+function usage() {
+    echo ""
+    echo "This is a script for bitacross-worker integration ts tests. Pass test name as first argument"
+    echo ""
+}
+
+[ $# -ne 1 ] && (usage; exit 1)
+TEST=$1
+
+echo "Using client binary $CLIENT_BIN"
+echo "Using node uri $NODEURL:$NPORT"
+echo "Using trusted-worker uri $WORKER1URL:$WORKER1PORT"
+echo "Using node http uri $NODEHTTPURL:$NPORT"
+echo ""
+
+cd /ts-tests
+pnpm install
+
+NODE_ENV=staging pnpm --filter integration-tests run test $TEST
diff --git a/bitacross-worker/docker/multiworker-docker-compose.yml b/bitacross-worker/docker/multiworker-docker-compose.yml
@@ -118,32 +118,6 @@ services:
       - "${SGX_QCNL:-/dev/null}:/etc/sgx_default_qcnl.conf"
     environment:
       - RUST_LOG=info,litentry_worker=debug,ws=warn,sp_io=error,substrate_api_client=warn,itc_parentchain_light_client=info,jsonrpsee_ws_client=warn,jsonrpsee_ws_server=warn,enclave_runtime=debug,ita_stf=debug,its_rpc_handler=warn,itc_rpc_client=warn,its_consensus_common=debug,its_state=warn,its_consensus_aura=warn,aura*=warn,its_consensus_slots=warn,itp_attestation_handler=debug,http_req=debug,lc_mock_server=warn,itc_rest_client=debug,lc_credentials=debug,lc_identity_verification=debug,lc_stf_task_receiver=debug,lc_stf_task_sender=debug,lc_data_providers=debug,itp_top_pool=debug,itc_parentchain_indirect_calls_executor=debug,
-      - TWITTER_OFFICIAL_URL=http://localhost:19527
-      - TWITTER_AUTH_TOKEN_V2=
-      - DISCORD_OFFICIAL_URL=http://localhost:19527
-      - LITENTRY_DISCORD_MICROSERVICE_URL=http://localhost:19527
-      - DISCORD_AUTH_TOKEN=
-      - ACHAINABLE_URL=http://localhost:19527
-      - ACHAINABLE_AUTH_KEY=
-      - CREDENTIAL_ENDPOINT=http://localhost:9933
-      - ONEBLOCK_NOTION_KEY=
-      - ONEBLOCK_NOTION_URL=http://localhost:19527
-      - SORA_QUIZ_MASTER_ID=SORA_QUIZ_MASTER_ID
-      - SORA_QUIZ_ATTENDEE_ID=SORA_QUIZ_ATTENDEE_ID
-      - MORALIS_API_URL=http://localhost:19527/moralis/
-      - MORALIS_SOLANA_API_URL=http://localhost:19527/moralis_solana/
-      - KARAT_DAO_API_URL=http://localhost:19527/karat_dao/
-      - MAGIC_CRAFT_API_URL=http://localhost:19527/magic_craft/
-      - MAGIC_CRAFT_API_KEY=
-      - MORALIS_API_KEY=
-      - NODEREAL_API_KEY=NODEREAL_API_KEY
-      - NODEREAL_API_URL=http://localhost:19527
-      - NODEREAL_API_CHAIN_NETWORK_URL=http://localhost:19527/nodereal_jsonrpc/
-      - GENIIDATA_URL=http://localhost:19527
-      - CONTEST_LEGEND_DISCORD_ROLE_ID=CONTEST_LEGEND_DISCORD_ROLE_ID
-      - CONTEST_POPULARITY_DISCORD_ROLE_ID=CONTEST_POPULARITY_DISCORD_ROLE_ID
-      - CONTEST_PARTICIPANT_DISCORD_ROLE_ID=CONTEST_PARTICIPANT_DISCORD_ROLE_ID
-      - BLOCKCHAIN_INFO_API_URL=http://localhost:19527/blockchain_info/
     networks:
       - litentry-test-network
     healthcheck:
@@ -176,32 +150,6 @@ services:
       - "${SGX_QCNL:-/dev/null}:/etc/sgx_default_qcnl.conf"
     environment:
       - RUST_LOG=info,litentry_worker=debug,ws=warn,sp_io=error,substrate_api_client=warn,itc_parentchain_light_client=info,jsonrpsee_ws_client=warn,jsonrpsee_ws_server=warn,enclave_runtime=debug,ita_stf=debug,its_rpc_handler=warn,itc_rpc_client=warn,its_consensus_common=debug,its_state=warn,its_consensus_aura=warn,aura*=warn,its_consensus_slots=warn,itp_attestation_handler=debug,http_req=debug,lc_mock_server=warn,itc_rest_client=debug,lc_credentials=debug,lc_identity_verification=debug,lc_stf_task_receiver=debug,lc_stf_task_sender=debug,lc_data_providers=debug,itp_top_pool=debug,itc_parentchain_indirect_calls_executor=debug,
-      - TWITTER_OFFICIAL_URL=http://localhost:19527
-      - TWITTER_AUTH_TOKEN_V2=
-      - DISCORD_OFFICIAL_URL=http://localhost:19527
-      - LITENTRY_DISCORD_MICROSERVICE_URL=http://localhost:19527
-      - DISCORD_AUTH_TOKEN=
-      - ACHAINABLE_URL=http://localhost:19527
-      - ACHAINABLE_AUTH_KEY=
-      - CREDENTIAL_ENDPOINT=http://localhost:9933
-      - ONEBLOCK_NOTION_KEY=
-      - ONEBLOCK_NOTION_URL=http://localhost:19527
-      - SORA_QUIZ_MASTER_ID=SORA_QUIZ_MASTER_ID
-      - SORA_QUIZ_ATTENDEE_ID=SORA_QUIZ_ATTENDEE_ID
-      - MORALIS_API_URL=http://localhost:19527/moralis/
-      - MORALIS_SOLANA_API_URL=http://localhost:19527/moralis_solana/
-      - KARAT_DAO_API_URL=http://localhost:19527/karat_dao/
-      - MAGIC_CRAFT_API_URL=http://localhost:19527/magic_craft/
-      - MAGIC_CRAFT_API_KEY=
-      - MORALIS_API_KEY=
-      - NODEREAL_API_KEY=NODEREAL_API_KEY
-      - NODEREAL_API_URL=http://localhost:19527
-      - NODEREAL_API_CHAIN_NETWORK_URL=http://localhost:19527/nodereal_jsonrpc/
-      - GENIIDATA_URL=http://localhost:19527
-      - CONTEST_LEGEND_DISCORD_ROLE_ID=CONTEST_LEGEND_DISCORD_ROLE_ID
-      - CONTEST_POPULARITY_DISCORD_ROLE_ID=CONTEST_POPULARITY_DISCORD_ROLE_ID
-      - CONTEST_PARTICIPANT_DISCORD_ROLE_ID=CONTEST_PARTICIPANT_DISCORD_ROLE_ID
-      - BLOCKCHAIN_INFO_API_URL=http://localhost:19527/blockchain_info/
     networks:
       - litentry-test-network
     healthcheck:
@@ -214,6 +162,38 @@ services:
       -u ws://litentry-node -U ws://bitacross-worker-2 -P 2011 -w 2101 -p 9912 -h 4645
       run --dev --skip-ra --request-state"
     restart: "no"
+  bitacross-worker-3:
+    image: litentry/bitacross-worker:latest
+    container_name: bitacross-worker-3
+    build:
+      context: ${PWD}/..
+      dockerfile: build.Dockerfile
+      target: deployed-worker
+    depends_on:
+      litentry-node:
+        condition: service_healthy
+      bitacross-worker-2:
+        condition: service_healthy
+    devices:
+      - "${SGX_PROVISION:-/dev/null}:/dev/sgx/provision"
+      - "${SGX_ENCLAVE:-/dev/null}:/dev/sgx/enclave"
+    volumes:
+      - "${AESMD:-/dev/null}:/var/run/aesmd"
+      - "${SGX_QCNL:-/dev/null}:/etc/sgx_default_qcnl.conf"
+    environment:
+      - RUST_LOG=info,litentry_worker=debug,ws=warn,sp_io=error,substrate_api_client=warn,itc_parentchain_light_client=info,jsonrpsee_ws_client=warn,jsonrpsee_ws_server=warn,enclave_runtime=debug,ita_stf=debug,its_rpc_handler=warn,itc_rpc_client=warn,its_consensus_common=debug,its_state=warn,its_consensus_aura=warn,aura*=warn,its_consensus_slots=warn,itp_attestation_handler=debug,http_req=debug,lc_mock_server=warn,itc_rest_client=debug,lc_credentials=debug,lc_identity_verification=debug,lc_stf_task_receiver=debug,lc_stf_task_sender=debug,lc_data_providers=debug,itp_top_pool=debug,itc_parentchain_indirect_calls_executor=debug,
+    networks:
+      - litentry-test-network
+    healthcheck:
+      test: curl -s -f http://bitacross-worker-3:4645/is_initialized || exit 1
+      interval: 30s
+      timeout: 10s
+      retries: 20
+    entrypoint:
+      "/usr/local/bin/bitacross-worker --clean-reset --ws-external -M bitacross-worker-3 -T wss://bitacross-worker-3
+      -u ws://litentry-node -U ws://bitacross-worker-3 -P 2011 -w 2101 -p 9912 -h 4645
+      run --dev --skip-ra --request-state"
+    restart: "no"
 volumes:
   ? relaychain-alice
   ? relaychain-bob