Add basic bitacross worker ci integration tests (#2934)

* rpc method for checking musig2 ceremony correctness

* test sign_bitcoin by enclave account

* ts code to test sign_bitcoin

* build bitacross docker image

* add bitacross integration tests

* enable enclave tests

* adjust test

* add reject unauthorized to ws client

* increase wait time for workers startup

* update filefilter

* trigger bitacross rebuild

* remove bitacross cache

Author: kziemianek

.github/file-filter.yml

@@ -47,7 +47,7 @@ bitacross_src: &bitacross_src
   - 'bitacross-worker/build.Dockerfile'
   - 'bitacross-worker/enclave-runtime/**'
 
-bitacross_test: &bitacross_src
+bitacross_test: &bitacross_test
   - 'bitacross-worker/ts-tests/**'
   - 'bitacross-worker/cli/*.sh'
   - 'docker/**'

.github/workflows/ci.yml

@@ -55,6 +55,11 @@ on:
         description: rebuild-tee-docker
         required: true
         default: true
+      rebuild-bitacross-docker:
+        type: boolean
+        description: rebuild-bitacross-docker
+        required: true
+        default: true
       push-docker:
         type: boolean
         description: push-docker
@@ -140,7 +145,7 @@ jobs:
           echo "push_docker=$push_docker" | tee -a $GITHUB_OUTPUT
           echo "run_parachain_test=$run_parachain_test" | tee -a $GITHUB_OUTPUT
           echo "run_tee_test=$run_tee_test" | tee -a $GITHUB_OUTPUT
-          echo "run_bitacross_test=$$run_tee_test" | tee -a $GITHUB_OUTPUT
+          echo "run_bitacross_test=$run_bitacross_test" | tee -a $GITHUB_OUTPUT
 
   fmt:
     runs-on: ubuntu-latest
@@ -511,6 +516,88 @@ jobs:
         if: failure()
         uses: andymckay/cancel-action@0.5
 
+  bitacross-build:
+    runs-on: ubuntu-latest
+    needs:
+      - fmt
+      - set-condition
+      - sequentialise
+    steps:
+      - name: Free up disk space
+        if: startsWith(runner.name, 'GitHub Actions')
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
+          swap-storage: false
+          large-packages: false
+
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          # use the docker driver to access the local image
+          # we don't need external caches or multi platforms here
+          # see https://docs.docker.com/build/drivers/
+          driver: docker
+
+      - name: Build local builder
+        if: needs.set-condition.outputs.rebuild_bitacross == 'true'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: bitacross-worker/build.Dockerfile
+          tags: local-builder:latest
+          target: builder
+          build-args: |
+            WORKER_MODE_ARG=offchain-worker            
+            ADDITIONAL_FEATURES_ARG=
+
+      - name: Build worker
+        if: needs.set-condition.outputs.rebuild_bitacross == 'true'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: bitacross-worker/build.Dockerfile
+          tags: litentry/bitacross-worker:latest
+          target: deployed-worker
+
+      - name: Build cli
+        if: needs.set-condition.outputs.rebuild_bitacross == 'true'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: bitacross-worker/build.Dockerfile
+          tags: litentry/bitacross-cli:latest
+          target: deployed-client
+
+      - name: Pull and tag worker and cli image optionally
+        if: needs.set-condition.outputs.rebuild_bitacross == 'false'
+        run: |
+          docker pull litentry/bitacross-worker:latest
+          docker pull litentry/bitacross-cli:latest
+
+      - run: docker images --all
+
+      - name: Test enclave
+        if: needs.set-condition.outputs.rebuild_bitacross == 'true'
+        # cargo test is not supported in the enclave
+        # see https://github.com/apache/incubator-teaclave-sgx-sdk/issues/232
+        run: docker run litentry/bitacross-worker:latest test --all
+
+      - name: Save docker images
+        run: docker save litentry/bitacross-worker:latest litentry/bitacross-cli:latest | gzip > litentry-bitacross.tar.gz
+
+      - name: Upload docker images
+        uses: actions/upload-artifact@v4
+        with:
+          name: litentry-bitacross
+          path: litentry-bitacross.tar.gz
+          if-no-files-found: error
+      - name: Fail early
+        if: failure()
+        uses: andymckay/cancel-action@0.5
+
   parachain-ts-test:
     runs-on: ubuntu-latest
     needs:
@@ -806,6 +893,82 @@ jobs:
           if-no-files-found: ignore
           retention-days: 3
 
+  bitacross-worker-test:
+    runs-on: ubuntu-latest
+    needs:
+      - set-condition
+      - parachain-build-dev
+      - bitacross-build
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - test_name: lit-sign-bitcoin
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Pull polkadot image
+        run: |
+          docker pull parity/polkadot
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: litentry-parachain-dev
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: litentry-bitacross
+
+      - name: Load docker image
+        run: |
+          docker load < litentry-parachain-dev.tar.gz
+          docker load < litentry-bitacross.tar.gz
+          docker images
+
+      - name: Enable corepack and pnpm
+        run: corepack enable && corepack enable pnpm
+
+      - name: Generate parachain artefacts
+        run: |
+          ./tee-worker/scripts/litentry/generate_parachain_artefacts.sh
+          ls -l docker/generated-rococo/
+          ls -l bitacross-worker/docker/litentry/
+
+      - name: Build litentry parachain docker images
+        run: |
+          cd tee-worker/docker
+          docker compose -f litentry-parachain.build.yml build
+
+      - name: Integration bitacross worker test ${{ matrix.test_name }}
+        if: needs.set-condition.outputs.run_bitacross_test == 'true'
+        timeout-minutes: 40
+        run: |
+          cd bitacross-worker/docker
+          docker compose -f multiworker-docker-compose.yml -f ${{ matrix.test_name }}.yml up --no-build --exit-code-from ${{ matrix.test_name }} ${{ matrix.test_name }}
+
+      - name: Stop integration multi worker docker containers
+        if: needs.set-condition.outputs.run_bitacross_test == 'true'
+        run: |
+          cd bitacross-worker/docker
+          docker compose -f multiworker-docker-compose.yml -f ${{ matrix.test_name }}.yml stop
+
+      - name: Collect docker logs if test fails
+        continue-on-error: true
+        uses: jwalton/gh-docker-logs@v2
+        if: failure()
+        with:
+          tail: all
+          dest: docker-logs
+
+      - name: Upload docker logs if test fails
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: ${{ matrix.test_name }}-docker-logs
+          path: docker-logs
+          if-no-files-found: ignore
+          retention-days: 3
+
   # Secrets are not passed to the runner when a workflow is triggered from a forked repository,
   # see https://docs.github.com/en/actions/security-guides/encrypted-secrets#using-encrypted-secrets-in-a-workflow
   #

bitacross-worker/bitacross/core/bc-task-processor/src/lib.rs

@@ -289,7 +289,7 @@ pub fn run_bit_across_handler_runner<SKR, SIGNINGAK, EKR, BKR, S, H, O, RRL, ERL
 
 	command_threads_pool.join();
 	event_threads_pool.join();
-	warn!("bit_across_task_receiver loop terminated");
+	warn!("bit_across_handler_runner loop terminated");
 }
 
 #[allow(clippy::type_complexity)]

bitacross-worker/build.Dockerfile

@@ -59,19 +59,8 @@ ENV SGX_COMMERCIAL_KEY=$SGX_COMMERCIAL_KEY
 WORKDIR $HOME/bitacross-worker
 COPY . $HOME
 
-RUN \
-  if [ "$IMAGE_FOR_RELEASE" = "true" ]; then \
-    echo "Omit cache for release image"; \
-    unset RUSTC_WRAPPER; \
-    make; \
-  else \
-    rm -rf /opt/rust/registry/cache && mv /home/ubuntu/worker-cache/registry/cache /opt/rust/registry && \
-    rm -rf /opt/rust/registry/index && mv /home/ubuntu/worker-cache/registry/index /opt/rust/registry && \
-    rm -rf /opt/rust/git/db && mv /home/ubuntu/worker-cache/git/db /opt/rust/git && \
-    rm -rf /opt/rust/sccache && mv /home/ubuntu/worker-cache/sccache /opt/rust && \
-    make && sccache --show-stats; \
-  fi
-
+RUN unset RUSTC_WRAPPER;
+RUN make
 RUN make mrenclave 2>&1 | grep MRENCLAVE | awk '{print $2}' > mrenclave.txt
 RUN cargo test --release
 

bitacross-worker/cli/lit_ts_integration_test.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# Copyright 2020-2024 Trust Computing GmbH.
+
+set -euo pipefail
+
+while getopts ":p:A:u:W:V:C:" opt; do
+    case $opt in
+        p)
+            NPORT=$OPTARG
+            ;;
+        A)
+            WORKER1PORT=$OPTARG
+            ;;
+        u)
+            NODEURL=$OPTARG
+            ;;
+        W)
+            NODEHTTPURL=$OPTARG
+            ;;
+        V)
+            WORKER1URL=$OPTARG
+            ;;
+        C)
+            CLIENT_BIN=$OPTARG
+            ;;
+    esac
+done
+
+# Using default port if none given as arguments.
+NPORT=${NPORT:-9912}
+NODEURL=${NODEURL:-"ws://litentry-node"}
+NODEHTTPURL=${NODEHTTPURL:-"http://litentry-node"}
+WORKER1PORT=${WORKER1PORT:-2011}
+WORKER1URL=${WORKER1URL:-"ws://bitacross-worker-1"}
+
+CLIENT_BIN=${CLIENT_BIN:-"/usr/local/bin/bitacross-cli"}
+
+CLIENT="${CLIENT_BIN} -p ${NPORT} -P ${WORKER1PORT} -u ${NODEURL} -U ${WORKER1URL}"
+
+function usage() {
+    echo ""
+    echo "This is a script for bitacross-worker integration ts tests. Pass test name as first argument"
+    echo ""
+}
+
+[ $# -ne 1 ] && (usage; exit 1)
+TEST=$1
+
+echo "Using client binary $CLIENT_BIN"
+echo "Using node uri $NODEURL:$NPORT"
+echo "Using trusted-worker uri $WORKER1URL:$WORKER1PORT"
+echo "Using node http uri $NODEHTTPURL:$NPORT"
+echo ""
+
+cd /ts-tests
+pnpm install
+
+NODE_ENV=staging pnpm --filter integration-tests run test $TEST

bitacross-worker/docker/lit-sign-bitcoin.yml

@@ -0,0 +1,27 @@
+services:
+    lit-sign-bitcoin:
+        image: litentry/bitacross-cli:latest
+        container_name: litentry-sign-bitcoin-test
+        volumes:
+            - ../ts-tests:/ts-tests
+            - ../cli:/usr/local/worker-cli
+        build:
+            context: ..
+            dockerfile: build.Dockerfile
+            target: deployed-client
+        depends_on:
+            litentry-node:
+                condition: service_healthy
+            bitacross-worker-1:
+                condition: service_healthy
+            bitacross-worker-2:
+                condition: service_healthy
+            bitacross-worker-3:
+                condition: service_healthy
+        networks:
+            - litentry-test-network
+        entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_integration_test.sh sign_bitcoin.test.ts 2>&1' "
+        restart: "no"
+networks:
+    litentry-test-network:
+        driver: bridge

bitacross-worker/docker/multiworker-docker-compose.yml

@@ -162,6 +162,38 @@ services:
       -u ws://litentry-node -U ws://bitacross-worker-2 -P 2011 -w 2101 -p 9912 -h 4645
       run --dev --skip-ra --request-state"
     restart: "no"
+  bitacross-worker-3:
+    image: litentry/bitacross-worker:latest
+    container_name: bitacross-worker-3
+    build:
+      context: ${PWD}/..
+      dockerfile: build.Dockerfile
+      target: deployed-worker
+    depends_on:
+      litentry-node:
+        condition: service_healthy
+      bitacross-worker-2:
+        condition: service_healthy
+    devices:
+      - "${SGX_PROVISION:-/dev/null}:/dev/sgx/provision"
+      - "${SGX_ENCLAVE:-/dev/null}:/dev/sgx/enclave"
+    volumes:
+      - "${AESMD:-/dev/null}:/var/run/aesmd"
+      - "${SGX_QCNL:-/dev/null}:/etc/sgx_default_qcnl.conf"
+    environment:
+      - RUST_LOG=info,litentry_worker=debug,ws=warn,sp_io=error,substrate_api_client=warn,itc_parentchain_light_client=info,jsonrpsee_ws_client=warn,jsonrpsee_ws_server=warn,enclave_runtime=debug,ita_stf=debug,its_rpc_handler=warn,itc_rpc_client=warn,its_consensus_common=debug,its_state=warn,its_consensus_aura=warn,aura*=warn,its_consensus_slots=warn,itp_attestation_handler=debug,http_req=debug,lc_mock_server=warn,itc_rest_client=debug,lc_credentials=debug,lc_identity_verification=debug,lc_stf_task_receiver=debug,lc_stf_task_sender=debug,lc_data_providers=debug,itp_top_pool=debug,itc_parentchain_indirect_calls_executor=debug,
+    networks:
+      - litentry-test-network
+    healthcheck:
+      test: curl -s -f http://bitacross-worker-3:4645/is_initialized || exit 1
+      interval: 30s
+      timeout: 10s
+      retries: 20
+    entrypoint:
+      "/usr/local/bin/bitacross-worker --clean-reset --ws-external -M bitacross-worker-3 -T wss://bitacross-worker-3
+      -u ws://litentry-node -U ws://bitacross-worker-3 -P 2011 -w 2101 -p 9912 -h 4645
+      run --dev --skip-ra --request-state"
+    restart: "no"
 volumes:
   ? relaychain-alice
   ? relaychain-bob

bitacross-worker/ts-tests/integration-tests/.env.local.example

@@ -0,0 +1,5 @@
+NODE_ENV = local
+WORKER_ENDPOINT = ws://localhost:2000
+NODE_ENDPOINT = ws://localhost:9944
+BINARY_DIR=../../bin
+LITENTRY_CLI_DIR=../../bin/bitacross-cli

bitacross-worker/ts-tests/integration-tests/.env.staging

@@ -0,0 +1,5 @@
+NODE_ENV = staging
+WORKER_ENDPOINT = ws://bitacross-worker-1:2011
+NODE_ENDPOINT = "ws://litentry-node:9912"
+BINARY_DIR=/usr/local/bin
+LITENTRY_CLI_DIR=/usr/local/bin/bitacross-cli