Improving sync parachain performance for workers #6913
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: General CI | |
| # This file is a joint CI of parachain and tee-worker, it contains: | |
| # - build (of docker images) | |
| # - format check | |
| # - unit tests | |
| # - integration tests | |
| # Some notes: | |
| # | |
| # [1] the tee-worker part is a modified version of tee-worker/.github/workflows/build_and_test.yml | |
| # with extra triggering control. | |
| # | |
| # [2] the original file (`tee-worker/.github/workflows/build_and_test.yml`) is kept to sync | |
| # upstream changes, therefore we need to manually apply the changes to this file. | |
| # [3] please beware that if a job in `needs` is skipped, its dependent job will also be skipped, | |
| # see https://github.com/actions/runner/issues/491 | |
| # | |
| # jobs that will always be executed: | |
| # - fmt | |
| # - set-condition | |
| # - parachain-build-dev | |
| # - identity-build | |
| # | |
| # [4] please note that job-level if `env` is not supported: | |
| # https://github.com/actions/runner/issues/1189 | |
| # as a workaround, we need to put it in a step-level or use `needs.outputs` | |
| # | |
| on: | |
| push: | |
| branches: | |
| - dev | |
| paths-ignore: | |
| - "**/dependabot.yml" | |
| - "**/README.md" | |
| pull_request: | |
| branches: | |
| - dev | |
| types: | |
| - opened | |
| - reopened | |
| - synchronize | |
| - ready_for_review | |
| workflow_dispatch: | |
| inputs: | |
| rebuild-parachain-docker: | |
| type: boolean | |
| description: rebuild-parachain-docker | |
| required: true | |
| default: true | |
| rebuild-tee-docker: | |
| type: boolean | |
| description: rebuild-tee-docker | |
| required: true | |
| default: true | |
| push-docker: | |
| type: boolean | |
| description: push-docker | |
| required: true | |
| default: false | |
| env: | |
| CARGO_TERM_COLOR: always | |
| DOCKER_BUILDKIT: 1 | |
| # the branch or tag on which this workflow is triggered | |
| # `head_ref` will only be set if the triggering event is `pull_request` | |
| REF_VERSION: ${{ github.head_ref || github.ref_name }} | |
| concurrency: | |
| # see https://stackoverflow.com/questions/74117321/if-condition-in-concurrency-in-gha | |
| # along with the `sequentialise` job below, it guarantees: | |
| # - for push in dev: all triggered CIs will be run sequentially, none is cancelled | |
| # - for PR: later triggered CIs will cancel previous runs, maximum one CI will be run | |
| group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| set-condition: | |
| runs-on: ubuntu-latest | |
| # see https://github.com/orgs/community/discussions/25722 | |
| if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }} | |
| outputs: | |
| rebuild_parachain: ${{ steps.env.outputs.rebuild_parachain }} | |
| rebuild_tee: ${{ steps.env.outputs.rebuild_tee }} | |
| rebuild_bitacross: ${{ steps.env.outputs.rebuild_bitacross }} | |
| push_docker: ${{ steps.env.outputs.push_docker }} | |
| run_parachain_test: ${{ steps.env.outputs.run_parachain_test }} | |
| run_tee_test: ${{ steps.env.outputs.run_tee_test }} | |
| run_bitacross_test: ${{ steps.env.outputs.run_bitacross_test }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # Checks to see if any files in the PR/commit match one of the listed file types. | |
| # We can use this filter to decide whether or not to build docker images | |
| - uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| filters: .github/file-filter.yml | |
| list-files: shell | |
| - name: Set condition | |
| id: env | |
| run: | | |
| rebuild_parachain=false | |
| rebuild_tee=false | |
| rebuild_bitacross=false | |
| push_docker=false | |
| run_parachain_test=false | |
| run_tee_test=false | |
| if [ "${{ github.event.inputs.rebuild-parachain-docker }}" = "true" ] || [ "${{ steps.filter.outputs.parachain_src }}" = "true" ]; then | |
| rebuild_parachain=true | |
| fi | |
| if [ "${{ github.event.inputs.rebuild-tee-docker }}" = "true" ] || [ "${{ steps.filter.outputs.tee_src }}" = "true" ]; then | |
| rebuild_tee=true | |
| fi | |
| if [ "${{ github.event.inputs.rebuild-bitacross-docker }}" = "true" ] || [ "${{ steps.filter.outputs.bitacross_src }}" = "true" ]; then | |
| rebuild_bitacross=true | |
| fi | |
| if [ "${{ github.event.inputs.push-docker }}" = "true" ]; then | |
| push_docker=true | |
| elif [ "${{ github.event_name }}" = 'push' ] && [ "${{ github.ref }}" = 'refs/heads/dev' ]; then | |
| push_docker=true | |
| fi | |
| if [ "${{ steps.filter.outputs.parachain_test }}" = "true" ] || [ "$rebuild_parachain" = "true" ]; then | |
| run_parachain_test=true | |
| fi | |
| if [ "${{ steps.filter.outputs.tee_test }}" = "true" ] || [ "$rebuild_parachain" = "true" ] || [ "$rebuild_tee" = "true" ]; then | |
| run_tee_test=true | |
| fi | |
| if [ "${{ steps.filter.outputs.bitacross_test }}" = "true" ] || [ "$rebuild_parachain" = "true" ] || [ "$rebuild_bitacross" = "true" ]; then | |
| run_bitacross_test=true | |
| fi | |
| echo "rebuild_parachain=$rebuild_parachain" | tee -a $GITHUB_OUTPUT | |
| echo "rebuild_tee=$rebuild_tee" | tee -a $GITHUB_OUTPUT | |
| echo "rebuild_bitacross=$rebuild_bitacross" | tee -a $GITHUB_OUTPUT | |
| echo "push_docker=$push_docker" | tee -a $GITHUB_OUTPUT | |
| echo "run_parachain_test=$run_parachain_test" | tee -a $GITHUB_OUTPUT | |
| echo "run_tee_test=$run_tee_test" | tee -a $GITHUB_OUTPUT | |
| echo "run_bitacross_test=$$run_tee_test" | tee -a $GITHUB_OUTPUT | |
| fmt: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install rust toolchain | |
| run: rustup show | |
| - name: Install pre-built taplo | |
| run: | | |
| mkdir -p $HOME/.local/bin | |
| wget -q https://github.com/tamasfe/taplo/releases/download/0.8.1/taplo-linux-x86_64.gz | |
| gzip -d taplo-linux-x86_64.gz | |
| cp taplo-linux-x86_64 $HOME/.local/bin/taplo | |
| chmod a+x $HOME/.local/bin/taplo | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - name: Parachain fmt check | |
| run: | | |
| cargo fmt --all -- --check | |
| taplo fmt --check | |
| - name: Tee-worker fmt check | |
| working-directory: ./tee-worker | |
| run: | | |
| cargo fmt --all -- --check | |
| taplo fmt --check | |
| - name: Tee-worker enclave-runtime fmt check | |
| working-directory: ./tee-worker/enclave-runtime | |
| run: | | |
| cargo fmt --all -- --check | |
| - name: bitacross-worker fmt check | |
| working-directory: ./bitacross-worker | |
| run: | | |
| cargo fmt --all -- --check | |
| taplo fmt --check | |
| - name: bitacross-worker enclave-runtime fmt check | |
| working-directory: ./bitacross-worker/enclave-runtime | |
| run: | | |
| cargo fmt --all -- --check | |
| - name: Enable corepack and pnpm | |
| run: corepack enable && corepack enable pnpm | |
| - name: Tee-worker ts-tests install npm deps | |
| working-directory: ./tee-worker/ts-tests | |
| # We can't update the lockfile here because the client-api hasn't been generated yet. | |
| run: pnpm install --frozen-lockfile | |
| - name: Tee-worker check ts code format | |
| working-directory: ./tee-worker/ts-tests | |
| run: pnpm run check-format | |
| - name: Fail early | |
| if: failure() | |
| uses: andymckay/cancel-action@0.5 | |
| # sequentialise the workflow runs on `dev` branch | |
| # the if condition is applied in step level to make this job always `successful` | |
| sequentialise: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Wait for previous run | |
| if: ${{ !failure() && (github.event_name == 'push') && (github.ref == 'refs/heads/dev') }} | |
| uses: litentry/consecutive-workflow-action@main | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| interval: 300 | |
| branch: dev | |
| parachain-clippy: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - fmt | |
| - set-condition | |
| - sequentialise | |
| if: needs.set-condition.outputs.rebuild_parachain == 'true' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install rust toolchain | |
| run: rustup show | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -yq openssl clang libclang-dev cmake protobuf-compiler | |
| - name: Run cargo clippy check | |
| run: make clippy | |
| - name: Fail early | |
| if: failure() | |
| uses: andymckay/cancel-action@0.5 | |
| tee-check: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - fmt | |
| - set-condition | |
| - sequentialise | |
| container: "litentry/litentry-tee-dev:latest" | |
| strategy: | |
| matrix: | |
| type: | |
| - tee # TODO: switch to identity | |
| - bitacross | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update && \ | |
| sudo apt-get install -yq openssl clang libclang-dev cmake protobuf-compiler | |
| - name: Cargo test | |
| working-directory: ./${{ matrix.type }}-worker | |
| run: | | |
| cargo test --release | |
| - name: Worker common clippy | |
| working-directory: ./${{ matrix.type }}-worker | |
| shell: bash | |
| run: | | |
| for d in . enclave-runtime; do | |
| pushd "$d" | |
| echo "::group::cargo clippy all" | |
| cargo clippy --release -- -D warnings | |
| echo "::endgroup::" | |
| echo "::group::cargo clippy development" | |
| cargo clippy --release --features development -- -D warnings | |
| echo "::endgroup::" | |
| echo "::group::cargo clippy offchain-worker" | |
| cargo clippy --release --features offchain-worker -- -D warnings | |
| echo "::group::cargo clippy offchain-worker,development" | |
| cargo clippy --release --features offchain-worker,development -- -D warnings | |
| echo "::endgroup::" | |
| popd | |
| done | |
| - name: Identity-worker specific clippy | |
| if: matrix.type == 'tee' | |
| working-directory: ./tee-worker | |
| shell: bash | |
| run: | | |
| for d in . enclave-runtime; do | |
| pushd "$d" | |
| echo "::group::cargo clippy sidechain" | |
| cargo clippy --release --features sidechain -- -D warnings | |
| echo "::endgroup::" | |
| echo "::group::cargo clippy evm" | |
| cargo clippy --release --features evm -- -D warnings | |
| echo "::endgroup::" | |
| echo "::group::cargo clippy sidechain development" | |
| cargo clippy --release --features sidechain,development -- -D warnings | |
| echo "::endgroup::" | |
| echo "::group::cargo clippy evm development" | |
| cargo clippy --release --features evm,development -- -D warnings | |
| echo "::endgroup::" | |
| popd | |
| done | |
| - name: Fail early | |
| if: failure() | |
| uses: andymckay/cancel-action@0.5 | |
| parachain-build-dev: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - fmt | |
| - set-condition | |
| - sequentialise | |
| steps: | |
| - name: Free up disk space | |
| if: startsWith(runner.name, 'GitHub Actions') | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| tool-cache: true | |
| swap-storage: false | |
| large-packages: false | |
| - uses: actions/checkout@v4 | |
| - name: Build docker image | |
| if: needs.set-condition.outputs.rebuild_parachain == 'true' | |
| run: | | |
| echo "::group::build docker image" | |
| ./scripts/build-docker.sh release latest --features=fast-runtime | |
| echo "::endgroup::" | |
| echo "::group::docker images" | |
| docker images --all | |
| echo "::endgroup::" | |
| - name: Pull docker image optionally | |
| if: needs.set-condition.outputs.rebuild_parachain == 'false' | |
| run: | | |
| docker pull litentry/litentry-parachain:latest | |
| docker pull litentry/litentry-chain-aio:latest | |
| - name: Save docker image | |
| run: | | |
| docker save litentry/litentry-parachain:latest litentry/litentry-chain-aio:latest | gzip > litentry-parachain-dev.tar.gz | |
| - name: Upload parachain docker image | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: litentry-parachain-dev | |
| path: litentry-parachain-dev.tar.gz | |
| if-no-files-found: error | |
| - name: Fail early | |
| if: failure() | |
| uses: andymckay/cancel-action@0.5 | |
| identity-build: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - fmt | |
| - set-condition | |
| - sequentialise | |
| steps: | |
| - name: Free up disk space | |
| if: startsWith(runner.name, 'GitHub Actions') | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| tool-cache: true | |
| swap-storage: false | |
| large-packages: false | |
| - uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| # use the docker driver to access the local image | |
| # we don't need external caches or multi platforms here | |
| # see https://docs.docker.com/build/drivers/ | |
| driver: docker | |
| - name: Cache worker-cache | |
| if: needs.set-condition.outputs.rebuild_tee == 'true' | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| worker-cache | |
| key: worker-cache-${{ env.REF_VERSION }}-${{ hashFiles('tee-worker/**/Cargo.lock', 'tee-worker/**/Cargo.toml') }} | |
| restore-keys: | | |
| worker-cache-${{ env.REF_VERSION }}- | |
| worker-cache- | |
| - name: Create cache folder if not exist | |
| if: needs.set-condition.outputs.rebuild_tee == 'true' | |
| run: | | |
| for i in 'git/db' 'registry/cache' 'registry/index' 'sccache'; do | |
| [ ! -d "worker-cache/$i" ] && mkdir -p "worker-cache/$i" || true | |
| echo "hello" > worker-cache/$i/nix | |
| done | |
| echo "::group::List worker-cache size" | |
| du -sh worker-cache/* | |
| echo "::endgroup::" | |
| echo "::group::Show disk usage" | |
| df -h . | |
| echo "::endgroup::" | |
| # cache mount in buildkit won't be exported as image layers, so it doesn't work well with GHA cache, see | |
| # https://github.com/moby/buildkit/issues/1512 | |
| # https://www.reddit.com/r/rust/comments/126xeyx/exploring_the_problem_of_faster_cargo_docker/ | |
| # https://hackmd.io/@kobzol/S17NS71bh (a great summary) | |
| # | |
| # the `reproducible-containers/buildkit-cache-dance` seems to be unstable too in my test | |
| # sometimes it works, sometimes it results in empty cache, root cause unclear | |
| # | |
| # thus the persistence of rust cache (w/ sccache) is maintained manually | |
| # | |
| # there's no cache-from/to as the docker image layer is too large and it takes too long to sync | |
| # moreoever, we are not so eager to have layer caches, as the most critical operation(cargo build) | |
| # is "atomic" and can't be broken into layers. | |
| - name: Build local builder | |
| if: needs.set-condition.outputs.rebuild_tee == 'true' | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: tee-worker/build.Dockerfile | |
| tags: local-builder:latest | |
| target: builder | |
| build-args: | | |
| WORKER_MODE_ARG=sidechain | |
| WORKER_ENV_DATA_PROVIDERS_CONFIG=1 | |
| WORKER_MOCK_SERVER=1 | |
| ADDITIONAL_FEATURES_ARG= | |
| - name: Copy caches from the built image | |
| if: needs.set-condition.outputs.rebuild_tee == 'true' | |
| run: | | |
| echo "::group::Show disk usage" | |
| df -h . | |
| echo "::endgroup::" | |
| echo "::group::docker images" | |
| docker images --all | |
| echo "::endgroup::" | |
| echo "::group::copy cache out" | |
| for i in 'git/db' 'registry/cache' 'registry/index'; do | |
| b="${i%/*}" | |
| rm -rf worker-cache/$i | |
| docker cp "$(docker create --rm local-builder:latest):/opt/rust/$i" worker-cache/$b | |
| done | |
| rm -rf worker-cache/sccache | |
| docker cp "$(docker create --rm local-builder:latest):/opt/rust/sccache" worker-cache | |
| du -sh worker-cache/* | |
| echo "::endgroup::" | |
| echo "::group::df -h ." | |
| df -h . | |
| echo "::endgroup::" | |
| - name: Build worker | |
| if: needs.set-condition.outputs.rebuild_tee == 'true' | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: tee-worker/build.Dockerfile | |
| tags: litentry/identity-worker:latest | |
| target: deployed-worker | |
| - name: Build cli | |
| if: needs.set-condition.outputs.rebuild_tee == 'true' | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: tee-worker/build.Dockerfile | |
| tags: litentry/identity-cli:latest | |
| target: deployed-client | |
| - name: Pull and tag worker and cli image optionally | |
| if: needs.set-condition.outputs.rebuild_tee == 'false' | |
| run: | | |
| docker pull litentry/identity-worker:latest | |
| docker pull litentry/identity-cli:latest | |
| - run: docker images --all | |
| - name: Test enclave | |
| if: needs.set-condition.outputs.rebuild_tee == 'true' | |
| # cargo test is not supported in the enclave | |
| # see https://github.com/apache/incubator-teaclave-sgx-sdk/issues/232 | |
| run: docker run litentry/identity-worker:latest test --all | |
| - name: Save docker images | |
| run: docker save litentry/identity-worker:latest litentry/identity-cli:latest | gzip > litentry-identity.tar.gz | |
| - name: Upload docker images | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: litentry-identity | |
| path: litentry-identity.tar.gz | |
| if-no-files-found: error | |
| - name: Fail early | |
| if: failure() | |
| uses: andymckay/cancel-action@0.5 | |
| parachain-ts-test: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - set-condition | |
| - parachain-build-dev | |
| strategy: | |
| matrix: | |
| chain: | |
| - litmus | |
| - litentry | |
| - rococo | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: litentry-parachain-dev | |
| - name: Load docker image | |
| run: | | |
| docker load < litentry-parachain-dev.tar.gz | |
| - name: Enable corepack and pnpm | |
| run: corepack enable && corepack enable pnpm | |
| - name: Run ts tests for ${{ matrix.chain }} | |
| if: needs.set-condition.outputs.run_parachain_test == 'true' | |
| timeout-minutes: 35 | |
| run: | | |
| make test-ts-docker-${{ matrix.chain }} | |
| - name: Collect docker logs if test fails | |
| continue-on-error: true | |
| uses: jwalton/gh-docker-logs@v2 | |
| if: failure() | |
| with: | |
| tail: all | |
| dest: docker-logs | |
| - name: Upload docker logs if test fails | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: ${{ matrix.chain }}-ts-tests-docker-logs | |
| path: docker-logs | |
| if-no-files-found: ignore | |
| retention-days: 3 | |
| - name: Archive logs if test fails | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: ${{ matrix.chain }}-ts-tests-artifact | |
| path: /tmp/parachain_dev/ | |
| if-no-files-found: ignore | |
| retention-days: 3 | |
| - name: Fail early | |
| if: failure() | |
| uses: andymckay/cancel-action@0.5 | |
| parachain-unit-test: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - fmt | |
| - set-condition | |
| - sequentialise | |
| # run_parachain_test is related to ts-tests only | |
| if: needs.set-condition.outputs.rebuild_parachain == 'true' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run all unittests | |
| run: | | |
| echo "::group::core-primitives unittest" | |
| cargo test --locked --release -p core-primitives --lib | |
| echo "::endgroup::" | |
| echo "::group::all pallets unittest" | |
| cargo test --locked --release -p pallet-* --lib | |
| echo "::endgroup::" | |
| echo "::group::all pallets unittest with runtime-benchmarks feature" | |
| cargo test --locked --release -p pallet-* --lib --features=runtime-benchmarks | |
| echo "::endgroup::" | |
| - name: Fail early | |
| if: failure() | |
| uses: andymckay/cancel-action@0.5 | |
| parachain-runtime-test: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - fmt | |
| - set-condition | |
| - sequentialise | |
| if: needs.set-condition.outputs.rebuild_parachain == 'true' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install toolchain | |
| run: rustup show | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update && \ | |
| sudo apt-get install -yq openssl clang libclang-dev cmake protobuf-compiler | |
| # We could have used matrix but the runtime tests are executed sequentially for a cleaner GHA visualisation graph. | |
| # It won't take much longer as we run them back to back. | |
| - name: Run runtime tests | |
| run: | | |
| echo "::group::rococo runtime test" | |
| cargo test --locked --release -p rococo-parachain-runtime --lib | |
| echo "::endgroup::" | |
| echo "::group::litmus runtime test" | |
| cargo test --locked --release -p litmus-parachain-runtime --lib | |
| echo "::endgroup::" | |
| echo "::group::litentry runtime test" | |
| cargo test --locked --release -p litentry-parachain-runtime --lib | |
| echo "::endgroup::" | |
| - name: Fail early | |
| if: failure() | |
| uses: andymckay/cancel-action@0.5 | |
| identity-single-worker-test: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - set-condition | |
| - parachain-build-dev | |
| - identity-build | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # Litentry | |
| - test_name: lit-di-substrate-identity-test | |
| - test_name: lit-di-evm-identity-test | |
| - test_name: lit-di-bitcoin-identity-test | |
| - test_name: lit-di-solana-identity-test | |
| - test_name: lit-di-vc-test | |
| - test_name: lit-dr-vc-test | |
| - test_name: lit-parentchain-nonce | |
| - test_name: lit-test-failed-parentchain-extrinsic | |
| - test_name: lit-twitter-identity-test | |
| - test_name: lit-discord-identity-test | |
| - test_name: lit-assertion-contracts-test | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Pull polkadot image | |
| run: | | |
| docker pull parity/polkadot | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: litentry-parachain-dev | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: litentry-identity | |
| - name: Load docker image | |
| run: | | |
| docker load < litentry-parachain-dev.tar.gz | |
| docker load < litentry-identity.tar.gz | |
| docker images | |
| - name: Enable corepack and pnpm | |
| run: corepack enable && corepack enable pnpm | |
| - name: Generate parachain artefacts | |
| run: | | |
| ./tee-worker/scripts/litentry/generate_parachain_artefacts.sh | |
| ls -l docker/generated-rococo/ | |
| ls -l tee-worker/docker/litentry/ | |
| - name: Build litentry parachain docker images | |
| run: | | |
| cd tee-worker/docker | |
| docker compose -f litentry-parachain.build.yml build | |
| - name: Integration single worker test ${{ matrix.test_name }} | |
| if: needs.set-condition.outputs.run_tee_test == 'true' | |
| timeout-minutes: 40 | |
| run: | | |
| cd tee-worker/docker | |
| docker compose -f docker-compose.yml -f ${{ matrix.test_name }}.yml up --no-build --exit-code-from ${{ matrix.test_name }} ${{ matrix.test_name }} | |
| - name: Stop integration single worker docker containers | |
| if: needs.set-condition.outputs.run_tee_test == 'true' | |
| run: | | |
| cd tee-worker/docker | |
| docker compose -f docker-compose.yml -f ${{ matrix.test_name }}.yml stop | |
| - name: Collect docker logs if test fails | |
| continue-on-error: true | |
| uses: jwalton/gh-docker-logs@v2 | |
| if: failure() | |
| with: | |
| tail: all | |
| dest: docker-logs | |
| - name: Upload docker logs if test fails | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: ${{ matrix.test_name }}-docker-logs | |
| path: docker-logs | |
| if-no-files-found: ignore | |
| retention-days: 3 | |
| identity-multi-worker-test: | |
| runs-on: ubuntu-latest | |
| continue-on-error: true | |
| needs: | |
| - set-condition | |
| - parachain-build-dev | |
| - identity-build | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - test_name: lit-di-bitcoin-identity-multiworker-test | |
| - test_name: lit-di-evm-identity-multiworker-test | |
| - test_name: lit-di-solana-identity-multiworker-test | |
| - test_name: lit-di-substrate-identity-multiworker-test | |
| - test_name: lit-di-vc-multiworker-test | |
| - test_name: lit-dr-vc-multiworker-test | |
| - test_name: lit-resume-worker | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Pull polkadot image | |
| run: | | |
| docker pull parity/polkadot | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: litentry-parachain-dev | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: litentry-identity | |
| - name: Load docker image | |
| run: | | |
| docker load < litentry-parachain-dev.tar.gz | |
| docker load < litentry-identity.tar.gz | |
| docker images | |
| - name: Enable corepack and pnpm | |
| run: corepack enable && corepack enable pnpm | |
| - name: Generate parachain artefacts | |
| run: | | |
| ./tee-worker/scripts/litentry/generate_parachain_artefacts.sh | |
| ls -l docker/generated-rococo/ | |
| ls -l tee-worker/docker/litentry/ | |
| - name: Build litentry parachain docker images | |
| run: | | |
| cd tee-worker/docker | |
| docker compose -f litentry-parachain.build.yml build | |
| - name: Integration multi worker test ${{ matrix.test_name }} | |
| if: needs.set-condition.outputs.run_tee_test == 'true' | |
| timeout-minutes: 40 | |
| run: | | |
| cd tee-worker/docker | |
| docker compose -f multiworker-docker-compose.yml -f ${{ matrix.test_name }}.yml up --no-build --exit-code-from ${{ matrix.test_name }} ${{ matrix.test_name }} | |
| - name: Stop integration multi worker docker containers | |
| if: needs.set-condition.outputs.run_tee_test == 'true' | |
| run: | | |
| cd tee-worker/docker | |
| docker compose -f multiworker-docker-compose.yml -f ${{ matrix.test_name }}.yml stop | |
| - name: Collect docker logs if test fails | |
| continue-on-error: true | |
| uses: jwalton/gh-docker-logs@v2 | |
| if: failure() | |
| with: | |
| tail: all | |
| dest: docker-logs | |
| - name: Upload docker logs if test fails | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: ${{ matrix.test_name }}-docker-logs | |
| path: docker-logs | |
| if-no-files-found: ignore | |
| retention-days: 3 | |
| # Secrets are not passed to the runner when a workflow is triggered from a forked repository, | |
| # see https://docs.github.com/en/actions/security-guides/encrypted-secrets#using-encrypted-secrets-in-a-workflow | |
| # | |
| # Only try to push docker image when | |
| # - parachain-ts-test passes | |
| # - identity-single-worker-test passes | |
| # - set-condition.outputs.push_docker is `true` | |
| # Whether the parachain or tee-worker image will actually be pushed still depends on if a new image was built/rebuilt. | |
| # This is important not to overwrite any other jobs where a rebuild **was** triggered. | |
| # | |
| # We don't have to depend on jobs like `parachain-unit-test` as they have the same trigger condition `rebuild_parachain`, | |
| # so there must be no new image if `parachain-unit-test` is skipped. | |
| # | |
| # `!failure()` needs to be used to cover skipped jobs | |
| push-docker: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - set-condition | |
| - parachain-ts-test | |
| - identity-single-worker-test | |
| if: ${{ !failure() && needs.set-condition.outputs.push_docker == 'true' }} | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: litentry-parachain-dev | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: litentry-identity | |
| - name: Load docker image | |
| run: | | |
| docker load < litentry-parachain-dev.tar.gz | |
| docker load < litentry-identity.tar.gz | |
| - name: Dockerhub login | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| # only push `litentry/litentry-parachain` if we rebuilt it | |
| - name: Push parachain image | |
| if: needs.set-condition.outputs.rebuild_parachain == 'true' | |
| run: | | |
| docker push litentry/litentry-parachain | |
| docker push litentry/litentry-chain-aio | |
| # only push TEE images if we rebuilt them | |
| - name: Push tee-worker image | |
| if: needs.set-condition.outputs.rebuild_tee == 'true' | |
| run: | | |
| docker push litentry/identity-worker | |
| docker push litentry/identity-cli |