diff --git a/.github/workflows/create-release-draft.yml b/.github/workflows/create-release-draft.yml index fa619fa504..62c3f510ef 100644 --- a/.github/workflows/create-release-draft.yml +++ b/.github/workflows/create-release-draft.yml @@ -24,10 +24,10 @@ on: required: true default: true release_tag: - description: an existing tag for creating release (e.g. v1.2.3) + description: an existing tag for creating release (e.g. p1.2.0-w0.0.1-101) required: true diff_tag: - description: an existing tag to run diff against (e.g. v1.2.0) + description: an existing tag to run diff against (e.g. p1.1.0-w0.0.1-100) default: "" required: false genesis_release: @@ -44,6 +44,7 @@ env: DIFF_TAG: ${{ github.event.inputs.diff_tag }} GENESIS_RELEASE: ${{ github.event.inputs.genesis_release }} DOCKER_BUILDKIT: 1 + REF_VERSION: ${{ github.head_ref || github.ref_name }} jobs: set-release-type: @@ -114,8 +115,8 @@ jobs: ${{ matrix.chain }}-parachain-srtool-digest.json ${{ matrix.chain }}-parachain-runtime.compact.compressed.wasm - ## build docker image of parachain binary ## - build-docker: + # build docker image of parachain binary ## + build-parachain-docker: if: ${{ github.event.inputs.release_client == 'true' }} runs-on: ubuntu-latest steps: @@ -127,7 +128,7 @@ jobs: - name: Set env run: | - DOCKER_TAG=$(echo ${{ env.RELEASE_TAG }} | cut -d'-' -f1 | sed 's/p/v/') + DOCKER_TAG=$(echo ${{ env.RELEASE_TAG }} | sed 's/p/v/;s/\(.*\)-w.*/\1/') echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV - name: Build docker image @@ -166,6 +167,121 @@ jobs: ${{ env.GENESIS_RELEASE }}-genesis-state ${{ env.GENESIS_RELEASE }}-genesis-wasm + build-worker-docker: + if: ${{ github.event.inputs.release_worker == 'true' }} + runs-on: ubuntu-latest + steps: + - name: Checkout codes on ${{ env.RELEASE_TAG }} + uses: actions/checkout@v4 + with: + ref: ${{ env.RELEASE_TAG }} + fetch-depth: 0 + - name: Set env + run: | + WORKER_TAG=$(echo ${{ env.RELEASE_TAG }} | sed 's/.*\(w.*\)/\1/;s/w/v/') + echo "WORKER_TAG=$WORKER_TAG" >> $GITHUB_ENV + + - name: Free up disk space + if: startsWith(runner.name, 'GitHub Actions') + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + swap-storage: false + large-packages: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + # use the docker driver to access the local image + # we don't need external caches or multi platforms here + # see https://docs.docker.com/build/drivers/ + driver: docker + + - name: Cache worker-cache + uses: actions/cache@v3 + with: + path: | + worker-cache + key: worker-cache-${{ env.REF_VERSION }}-${{ hashFiles('tee-worker/**/Cargo.lock', 'tee-worker/**/Cargo.toml') }} + restore-keys: | + worker-cache-${{ env.REF_VERSION }}- + worker-cache- + + - name: Create cache folder if not exist + run: | + for i in 'git/db' 'registry/cache' 'registry/index' 'sccache'; do + [ ! -d "worker-cache/$i" ] && mkdir -p "worker-cache/$i" || true + echo "hello" > worker-cache/$i/nix + done + echo "::group::List worker-cache size" + du -sh worker-cache/* + echo "::endgroup::" + echo "::group::Show disk usage" + df -h . + echo "::endgroup::" + + - name: Build local builder + uses: docker/build-push-action@v5 + with: + context: . + file: tee-worker/build.Dockerfile + tags: local-builder:latest + target: builder + build-args: | + WORKER_MODE_ARG=sidechain + ADDITIONAL_FEATURES_ARG= + + - name: Copy caches from the built image + run: | + echo "::group::Show disk usage" + df -h . + echo "::endgroup::" + echo "::group::docker images" + docker images --all + echo "::endgroup::" + echo "::group::copy cache out" + for i in 'git/db' 'registry/cache' 'registry/index'; do + b="${i%/*}" + rm -rf worker-cache/$i + docker cp "$(docker create --rm local-builder:latest):/opt/rust/$i" worker-cache/$b + done + rm -rf worker-cache/sccache + docker cp "$(docker create --rm local-builder:latest):/opt/rust/sccache" worker-cache + du -sh worker-cache/* + echo "::endgroup::" + echo "::group::df -h ." + df -h . + echo "::endgroup::" + + - name: Build worker + uses: docker/build-push-action@v5 + with: + context: . + file: tee-worker/build.Dockerfile + tags: litentry/litentry-worker:${{ env.WORKER_TAG }} + target: deployed-worker + + - name: Build cli + uses: docker/build-push-action@v5 + with: + context: . + file: tee-worker/build.Dockerfile + tags: litentry/litentry-cli:${{ env.WORKER_TAG }} + target: deployed-client + + - run: docker images --all + + - name: Dockerhub login + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Push worker image + run: | + docker push litentry/litentry-worker:$WORKER_TAG + docker push litentry/litentry-cli:$WORKER_TAG + ## Build the enclave and package config files build-tee: if: ${{ github.event.inputs.release_worker == 'true' }} || ${{ github.event.inputs.release_enclave == 'true' }} @@ -181,8 +297,8 @@ jobs: ref: ${{ env.RELEASE_TAG }} fetch-depth: 0 - - name: Build release artefacts - run: | + - name: Build release artefacts + run: | source /opt/intel/sgxsdk/environment ./tee-worker/scripts/litentry/release/build.sh ${{ github.event.inputs.release_worker }} ${{ github.event.inputs.release_enclave }} @@ -214,11 +330,11 @@ jobs: - name: Fail early if: failure() uses: andymckay/cancel-action@0.3 - + ## test again the built docker image ## run-ts-tests: runs-on: ubuntu-latest - needs: build-docker + needs: build-parachain-docker strategy: matrix: chain: @@ -235,7 +351,7 @@ jobs: - name: Download and tag docker image run: | - export DOCKER_TAG=$(echo ${{ env.RELEASE_TAG }} | cut -d'-' -f1 | sed 's/p/v/') + export DOCKER_TAG=$(echo ${{ env.RELEASE_TAG }} | sed 's/p/v/;s/\(.*\)-w.*/\1/') docker pull litentry/litentry-parachain:$DOCKER_TAG docker tag litentry/litentry-parachain:$DOCKER_TAG litentry/litentry-parachain:latest @@ -260,7 +376,7 @@ jobs: ## check extrinsic ## extrinsic-ordering-check-from-bin: runs-on: ubuntu-latest - needs: build-docker + needs: build-parachain-docker strategy: matrix: chain: [rococo, litmus, litentry] @@ -280,7 +396,7 @@ jobs: - name: Prepare output and compare the metadata timeout-minutes: 3 run: | - export DOCKER_TAG=$(echo ${{ env.RELEASE_TAG }} | cut -d'-' -f1 | sed 's/p/v/') + export DOCKER_TAG=$(echo ${{ env.RELEASE_TAG }} | sed 's/p/v/;s/\(.*\)-w.*/\1/') PARACHAIN_NAME=local-parachain BASE_URL=ws://127.0.0.1:9944 chain=${{ matrix.chain }} @@ -310,16 +426,16 @@ jobs: uses: actions-cool/issues-helper@v3 id: findissueid with: - actions: 'find-issues' + actions: "find-issues" token: ${{ secrets.GITHUB_TOKEN }} - issue-state: 'open' + issue-state: "open" title-includes: Litentry-parachain ${{ env.RELEASE_TAG }} Release checklist - + - name: Create comment if: ${{ steps.findissueid.outputs.issues }} != '[]' uses: actions-cool/issues-helper@v3 with: - actions: 'create-comment' + actions: "create-comment" token: ${{ secrets.GITHUB_TOKEN }} issue-number: ${{ fromJson(steps.findissueid.outputs.issues)[0].number }} body: | @@ -334,7 +450,7 @@ jobs: - set-release-type - build-tee - run-ts-tests - - build-wasm + - build-wasm if: | !failure() && (success('build-wasm') || success('run-ts-tests') || success('build-tee')) @@ -347,7 +463,7 @@ jobs: - name: Download all artefacts uses: actions/download-artifact@v3 - + - name: Generate release notes run: | export MRENCLAVE="${{ needs.build-tee.outputs.mrenclave }}" diff --git a/.github/workflows/release-ts-api-package.yml b/.github/workflows/release-ts-api-package.yml index 5505c02816..b61bc5d52a 100644 --- a/.github/workflows/release-ts-api-package.yml +++ b/.github/workflows/release-ts-api-package.yml @@ -3,19 +3,10 @@ name: Release Ts API Package on: workflow_dispatch: - inputs: - parachain-tag: - description: 'Parachain docker image tag' - required: true - default: 'latest' - worker-tag: - description: 'Worker docker image tag' - required: true - default: 'latest' - release-tag: - description: 'Client-api release tag' - required: true - default: 'latest' + inputs: + release-tag: + description: "Client-api release tag (e.g. p1.2.0-9701-w0.0.1-101)" + required: true env: NODE_AUTH_TOKEN: ${{ secrets.RELEASE_TS_API_PACKAGE_TOKEN }} @@ -25,18 +16,24 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set ENV + run: | + # extracting parachain version and worker version from release tag + echo "PARACHAIN_TAG=$(echo ${{inputs.release-tag}} | sed 's/p/v/;s/\(.*\)-w.*/\1/')" >> $GITHUB_ENV + echo "WORKER_TAG=$(echo ${{inputs.release-tag}} | sed 's/.*\(w.*\)/\1/;s/w/v/')" >> $GITHUB_ENV + - name: Pull litentry image optionally run: | docker pull parity/polkadot - docker pull litentry/litentry-worker:${{ inputs.worker-tag }} - docker pull litentry/litentry-cli:${{ inputs.worker-tag }} - docker pull litentry/litentry-parachain:${{ inputs.parachain-tag }} + docker pull litentry/litentry-worker:$WORKER_TAG + docker pull litentry/litentry-cli:$WORKER_TAG + docker pull litentry/litentry-parachain:$PARACHAIN_TAG - name: Re-tag docker image run: | - docker tag litentry/litentry-worker:${{ inputs.worker-tag }} litentry/litentry-worker:latest - docker tag litentry/litentry-cli:${{ inputs.worker-tag }} litentry/litentry-cli:latest - docker tag litentry/litentry-parachain:${{ inputs.parachain-tag }} litentry/litentry-parachain:latest + docker tag litentry/litentry-worker:$WORKER_TAG litentry/litentry-worker:latest + docker tag litentry/litentry-cli:$WORKER_TAG litentry/litentry-cli:latest + docker tag litentry/litentry-parachain:$PARACHAIN_TAG litentry/litentry-parachain:latest - run: docker images --all @@ -51,9 +48,8 @@ jobs: run: | cd tee-worker/docker docker compose -f litentry-parachain.build.yml build - + - name: Update metadata and generate types - timeout-minutes: 10 run: | cd tee-worker/docker docker compose -f docker-compose.yml -f lit-ts-api-package-build.yml up --no-build --exit-code-from lit-ts-api-package-build lit-ts-api-package-build @@ -83,7 +79,6 @@ jobs: echo "$api dist and build files do not exist. Publishing failed." exit 1 fi - npm publish --tag ${{ inputs.release-tag }} echo "------------------------$api published------------------------" @@ -109,4 +104,4 @@ jobs: with: name: logs-lit-ts-api-package-build path: logs - if-no-files-found: ignore \ No newline at end of file + if-no-files-found: ignore diff --git a/tee-worker/cli/lit_ts_api_package_build.sh b/tee-worker/cli/lit_ts_api_package_build.sh index 0d962815eb..684833b7c5 100755 --- a/tee-worker/cli/lit_ts_api_package_build.sh +++ b/tee-worker/cli/lit_ts_api_package_build.sh @@ -42,8 +42,8 @@ echo "Using client binary $CLIENT_BIN" echo "Using node uri $NODEURL:$NPORT" echo "Using trusted-worker uri $WORKER1URL:$WORKER1PORT" echo "Using node http uri $NODEHTTPURL:$NPORT" -echo "" - +echo "waiting 20 secs worker to run successfully" +sleep 20 cd /client-api/parachain-api curl -s -H "Content-Type: application/json" -d '{"id": "1", "jsonrpc": "2.0", "method": "state_getMetadata", "params": []}' $NODEHTTPURL:$NPORT > prepare-build/litentry-parachain-metadata.json echo "update parachain metadata" diff --git a/tee-worker/docker/lit-ts-api-package-build.yml b/tee-worker/docker/lit-ts-api-package-build.yml index 979cc7205e..21fcd8e135 100644 --- a/tee-worker/docker/lit-ts-api-package-build.yml +++ b/tee-worker/docker/lit-ts-api-package-build.yml @@ -13,11 +13,13 @@ services: litentry-node: condition: service_healthy litentry-worker-1: - condition: service_healthy + # using +service_started+ over +service_healthy+ since worker runs successfully but can not connect to parachain + # as requires additional pre-setup for parachain image which built in production mode + # for generating types there is no need for fully workable interaction between worker and parachain + condition: service_started networks: - litentry-test-network - entrypoint: - "/usr/local/worker-cli/lit_ts_api_package_build.sh -p 9912 -u ws://litentry-node + entrypoint: "/usr/local/worker-cli/lit_ts_api_package_build.sh -p 9912 -u ws://litentry-node -W http://litentry-node -V wss://litentry-worker-1 -A 2011 -C /usr/local/bin/litentry-cli 2>&1" restart: "no" networks: