pvf reuse process #6852
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # Copyright Quadrivium LLC | |
| # All Rights Reserved | |
| # SPDX-License-Identifier: Apache-2.0 | |
| # | |
| name: Build | |
| on: | |
| push: | |
| branches: [ master ] | |
| tags: [ '*' ] | |
| pull_request: | |
| workflow_dispatch: | |
| env: | |
| BUILD_DIR: build | |
| KAGOME_IN_DOCKER: 1 | |
| CTEST_OUTPUT_ON_FAILURE: 1 | |
| GITHUB_HUNTER_USERNAME: ${{ secrets.HUNTER_USERNAME }} | |
| GITHUB_HUNTER_TOKEN: ${{ secrets.HUNTER_TOKEN }} | |
| # Change all container: values to this variable once this feature is available | |
| # https://github.community/t/how-to-set-and-access-a-workflow-variable/17335/6 | |
| CACHE_VERSION: v07 | |
| CACHE_PATHS: | | |
| ~/Library/Caches/pip | |
| ~/.cargo | |
| ~/.ccache | |
| ~/.hunter | |
| ~/.rustup | |
| jobs: | |
| MacOS: | |
| runs-on: macos-14 | |
| timeout-minutes: 120 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CACHE_PATHS }} | |
| key: ${{ github.job }}-${{ env.CACHE_VERSION }} | |
| - name: install | |
| run: ./housekeeping/macos/dependency.sh | |
| env: | |
| KAGOME_MAC_CI: 1 | |
| - name: build | |
| run: ./housekeeping/make_build.sh -G Ninja -DCLEAR_OBJS=ON -DCOVERAGE=OFF -DWASM_COMPILER=WasmEdge -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/cxx20.cmake | |
| env: | |
| KAGOME_IN_DOCKER: 0 | |
| KAGOME_MAC_CI: 1 | |
| Linux: | |
| if: ${{ !(github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/' )) }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| options: | |
| - name: "Linux: clang-16 External Project" | |
| run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-16_cxx20.cmake | |
| name: "${{ matrix.options.name }}" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| container: qdrvm/kagome-dev:9-minideb | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CACHE_PATHS }} | |
| key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} | |
| - name: df du | |
| run: | | |
| df -m || true | |
| du -hd1 /__w /github || true | |
| - name: Install mold | |
| run: ./housekeeping/ci_install_mold.sh --make-default | |
| - name: "${{ matrix.options.name }}" | |
| run: "${{ matrix.options.run }}" | |
| - name: df du | |
| if: ${{ always() }} | |
| run: | | |
| df -m || true | |
| du -hd1 /__w /github || true | |
| Linux-self-hosted: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| options: | |
| - name: "Self-hosted: Linux: gcc-12 ASAN" | |
| run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-12_cxx20.cmake -DASAN=ON | |
| - name: "Self-hosted: Linux: clang-16 TSAN WAVM" | |
| run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-16_cxx20.cmake -DTSAN=ON -DWASM_COMPILER=WAVM | |
| - name: "Self-hosted: Linux: clang-16 UBSAN" | |
| run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-16_cxx20.cmake -DUBSAN=ON | |
| - name: "Self-hosted: Linux: clang-16 External Project" | |
| run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-16_cxx20.cmake | |
| name: "${{ matrix.options.name }}" | |
| runs-on: [ actions-runner-controller ] | |
| timeout-minutes: 120 | |
| container: qdrvm/kagome-dev:9-minideb | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CACHE_PATHS }} | |
| key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} | |
| - name: Install mold | |
| run: ./housekeeping/ci_install_mold.sh --make-default | |
| - name: "${{ matrix.options.name }}" | |
| run: "${{ matrix.options.run }}" | |
| clang-tidy: | |
| name: "Linux: clang-tidy" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| container: qdrvm/kagome-dev:9-minideb | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CACHE_PATHS }} | |
| key: ${{ github.job }}-${{ env.CACHE_VERSION }} | |
| - name: clang-tidy | |
| env: | |
| # build only generated files, so clang-tidy will work correctly | |
| BUILD_FINAL_TARGET: generated | |
| # run build for both WAVM and WasmEdge to download both, otherwise clang-tidy | |
| # isn't able to find headers | |
| run: | | |
| ./housekeeping/make_build.sh | |
| ./housekeeping/clang-tidy-diff.sh | |
| coverage-self-hosted: | |
| if: false # ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/') || contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted') }} | |
| name: "Self-hosted: Linux: gcc-12 coverage/sonar" | |
| runs-on: [ actions-runner-controller ] | |
| timeout-minutes: 120 | |
| container: qdrvm/kagome-dev:9-minideb | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CACHE_PATHS }} | |
| key: ${{ github.job }}-${{ env.CACHE_VERSION }} | |
| - name: Install mold | |
| run: ./housekeeping/ci_install_mold.sh --make-default | |
| - name: makeBuild | |
| env: | |
| BUILD_FINAL_TARGET: ctest_coverage | |
| run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-12_cxx20.cmake -DCOVERAGE=ON -DCLEAR_OBJS=ON | |
| - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} | |
| name: Submit Coverage | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| run: if [ "CODECOV_TOKEN" != "null" ]; then ./housekeeping/codecov.sh; else echo "Some secret undefined. Step passed..."; fi | |
| - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} | |
| name: Sonar | |
| env: | |
| SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} | |
| GITHUB_USERNAME: ${{ secrets.GITHUB_USERNAME }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| BRANCH_NAME: ${{ github.ref }} | |
| run: if [ "$SONAR_TOKEN" != "null" -a "$GITHUB_USERNAME" != "null" -a "$GITHUB_TOKEN" != "null" ]; then ./housekeeping/sonar.sh; else echo "Some secret undefined. Step passed..."; fi | |
| Minideb: | |
| if: ${{ !(github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/' )) }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| options: | |
| - name: "Minideb: Build Debug" | |
| build-type: "Debug" | |
| - name: "Minideb: Build Release" | |
| build-type: "Release" | |
| name: "${{ matrix.options.name }}" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| container: qdrvm/kagome-dev:9-minideb | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CACHE_PATHS }} | |
| key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} | |
| - name: "${{ matrix.options.name }}" | |
| env: | |
| BUILD_TYPE: "${{ matrix.options.build-type }}" | |
| run: ./housekeeping/docker/kagome-dev/make.sh | |
| Minideb-self-hosted: | |
| if: ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/') || contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted') }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| options: | |
| - name: "Self-hosted: Minideb: Build Debug" | |
| build-type: "Debug" | |
| - name: "Self-hosted: Minideb: Build Release" | |
| build-type: "Release" | |
| name: "${{ matrix.options.name }}" | |
| runs-on: [ actions-runner-controller ] | |
| timeout-minutes: 120 | |
| container: qdrvm/kagome-dev:9-minideb | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CACHE_PATHS }} | |
| key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} | |
| - name: "${{ matrix.options.name }}" | |
| env: | |
| BUILD_TYPE: "${{ matrix.options.build-type }}" | |
| run: ./housekeeping/docker/kagome-dev/make.sh | |
| push-self-hosted: | |
| if: ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/' ) }} | |
| needs: [clang-tidy, MacOS, Minideb-self-hosted] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| options: | |
| - name: "Self-hosted: Push Debug Docker image" | |
| build-type: "Debug" | |
| - name: "Self-hosted: Push Release Docker image" | |
| build-type: "Release" | |
| - name: "Self-hosted: Push RelWithDebInfo Docker image" | |
| build-type: "RelWithDebInfo" | |
| runs-on: [ actions-runner-controller ] | |
| timeout-minutes: 120 | |
| container: qdrvm/kagome-dev:9-minideb | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CACHE_PATHS }} | |
| key: ${{ github.job }}-${{ env.CACHE_VERSION }} | |
| - run: git config --global --add safe.directory /__w/kagome/kagome | |
| - run: git fetch --prune --unshallow | |
| - name: build | |
| env: | |
| BUILD_TYPE: "${{ matrix.options.build-type }}" | |
| run: ./housekeeping/docker/kagome-dev/make.sh | |
| - uses: azure/docker-login@83efeb77770c98b620c73055fbb59b2847e17dc0 | |
| with: | |
| login-server: https://index.docker.io/v1/ | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_TOKEN }} | |
| - name: docker pack and push | |
| id: docker-pack-and-push | |
| env: | |
| VERSION: ${{ github.ref }} | |
| BUILD_TYPE: "${{ matrix.options.build-type }}" | |
| run: | | |
| ./housekeeping/docker/kagome/build_and_push.sh | |
| - name: Save TAG | |
| id: save-tag | |
| if: matrix.options.build-type == 'Release' | |
| run: TAG=$(cat tag_output.txt) && echo "tag=$TAG" >> $GITHUB_OUTPUT | |
| outputs: | |
| release-tag: ${{ steps.save-tag.outputs.tag }} | |
| Prepare-Zombie-Tests-Docker: | |
| runs-on: [ actions-runner-controller ] | |
| timeout-minutes: 60 | |
| needs: [push-self-hosted] | |
| steps: | |
| - uses: azure/docker-login@83efeb77770c98b620c73055fbb59b2847e17dc0 | |
| with: | |
| login-server: https://index.docker.io/v1/ | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_TOKEN }} | |
| - name: Pull Kagome Docker Image | |
| run: docker pull ${{ needs.push-self-hosted.outputs.release-tag }} | |
| - name: Load image | |
| run: docker image ls -a | |
| - name: Set owner of working dir recursively | |
| run: sudo chown -R $(whoami) . | |
| - uses: actions/checkout@v4 | |
| - name: Extract branch name | |
| shell: bash | |
| run: | | |
| branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} | |
| echo "branch=${branch}" >> $GITHUB_OUTPUT | |
| echo "Branch name: ${branch}" | |
| id: extract_branch | |
| - name: Build zombie-tester | |
| working-directory: ./zombienet/docker | |
| run: make tester BRANCH_NAME=${{ steps.extract_branch.outputs.branch }} KAGOME_IMAGE=${{ needs.push-self-hosted.outputs.release-tag }} | |
| - name: Archive zombie-tester | |
| shell: bash | |
| run: docker save -o /tmp/zombie-tester.tar zombie-tester:latest | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: zombie-tester | |
| path: /tmp/zombie-tester.tar | |
| Zombie-Tests: | |
| runs-on: [ actions-runner-controller ] | |
| timeout-minutes: 60 | |
| needs: [Prepare-Zombie-Tests-Docker] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| options: | |
| - name: "PVF preparation & execution time" | |
| test: "test-polkadot-functional-0001-parachains-pvf" | |
| - name: "Disputes initiation, conclusion and lag" | |
| test: "test-polkadot-functional-0002-parachains-disputes" | |
| - name: "BEEFY voting and finality, test MMR proofs. Assumes Rococo sessions of 1 minute" | |
| test: "test-polkadot-functional-0003-beefy-and-mmr" | |
| - name: "Dispute finality lag when 1/3 of parachain validators always attempt to include an invalid block" | |
| test: "test-polkadot-functional-0004-parachains-garbage-candidate" | |
| - name: "Past-session dispute slashing" | |
| test: "test-polkadot-functional-0005-parachains-disputes-past-session" | |
| - name: "Test if parachains make progress with most of approvals being tranch0" | |
| test: "test-polkadot-functional-0006-parachains-max-tranche0" | |
| - name: "Test if disputes triggered on finalized blocks within scope always end as valid" | |
| test: "test-polkadot-functional-0007-dispute-freshly-finalized" | |
| - name: "Test if disputes triggered on finalized blocks out of scope never get to be confirmed and concluded" | |
| test: "test-polkadot-functional-0008-dispute-old-finalized" | |
| - name: "Approval voting coalescing does not lag finality" | |
| test: "test-polkadot-functional-0009-approval-voting-coalescing" | |
| - name: "Test validator disabling effects" | |
| test: "test-polkadot-functional-0010-validator-disabling" | |
| - name: "Test we are producing blocks at 6 seconds clip" | |
| test: "test-polkadot-functional-0011-async-backing-6-seconds-rate" | |
| steps: | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: zombie-tester | |
| path: /tmp | |
| - name: Load image | |
| run: | | |
| docker load --input /tmp/zombie-tester.tar | |
| docker image ls -a | |
| - uses: actions/checkout@v4 | |
| - name: Run test | |
| working-directory: ./zombienet/docker | |
| run: make ${{ matrix.options.test }} ZOMBIE_TESTER_IMAGE=zombie-tester:latest | |
| - name: Upload test logs as artifact | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: logs-${{ matrix.options.test }} | |
| path: /tmp/test_logs |