Integrate risc0-ethereum on CI

.github/cargo_local_patch.py

@@ -0,0 +1,109 @@
+import argparse
+import toml
+import os
+import fnmatch
+
+def find_cargo_toml_files(start_path):
+    matches = []
+    for root, _, filenames in os.walk(start_path):
+        for filename in fnmatch.filter(filenames, 'Cargo.toml'):
+            matches.append(os.path.join(root, filename))
+    return matches
+
+def is_subdirectory(path, of_directory):
+    absolute_path = os.path.abspath(path)
+    absolute_of_directory = os.path.abspath(of_directory)
+    return absolute_path.startswith(absolute_of_directory)
+
+def modify_dependency(details, new_path, start_directory):
+    # If details is a string, it's in the simple format. Convert it to a dictionary.
+    if isinstance(details, str):
+        details = {'version': details}  # Convert to dictionary with version key
+    # Now details is guaranteed to be a dictionary
+    # Skip if the depdendency uses the workspace definition
+    if details.get('workspace') is not True:
+        current_path = details.get('path')
+        if current_path and is_subdirectory(current_path, start_directory):
+            # If the current path is a subdirectory of the start_directory, do not modify it.
+            return details
+
+        details['path'] = new_path
+        details.pop('version', None)
+        details.pop('git', None)
+        details.pop('rev', None)
+        details.pop('branch', None)
+    return details
+
+# Modify the list of dependencies in-place, setting a path reference for those in the mapping.
+def process_dependencies(dependencies, base_path, dep_path_mapping, start_directory):
+    for dep, details in dependencies.items():
+        # Check if a dep starts with "risc0-" or "bonsai-" and is not in the mapping.
+        # This should make sure to patch all the dependencies or fail if new are found.
+        if (dep.startswith("risc0-") or dep.startswith("bonsai-")) and dep not in dep_path_mapping:
+            raise ValueError(f"Dependency '{dep}' starts with 'risc0-' or 'bonsai-' but is not in dep_path_mapping.")
+        if dep in dep_path_mapping:
+            new_path = os.path.join(base_path, dep_path_mapping[dep])
+            if not os.path.exists(new_path):
+                raise FileNotFoundError(f"The path {new_path} does not exist.")
+            new_details = modify_dependency(details, new_path, start_directory)
+            dependencies[dep] = new_details
+
+def process_cargo_file(cargo_toml_path, dep_path_mapping, start_directory):
+    base_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../")
+
+    with open(cargo_toml_path, 'r') as file:
+        data = toml.load(file)
+
+    # Process regular dependencies
+    if 'dependencies' in data:
+        process_dependencies(data['dependencies'], base_path, dep_path_mapping, start_directory)
+    # Process development dependencies
+    if 'dev-dependencies' in data:
+        process_dependencies(data['dev-dependencies'], base_path, dep_path_mapping, start_directory)
+    # Process build dependencies
+    if 'build-dependencies' in data:
+        process_dependencies(data['build-dependencies'], base_path, dep_path_mapping, start_directory)
+    # Process workspace dependencies, if present
+    if 'workspace' in data and 'dependencies' in data['workspace']:
+        process_dependencies(data['workspace']['dependencies'], base_path, dep_path_mapping, start_directory)
+
+    with open(cargo_toml_path, 'w') as file:
+        toml.dump(data, file)
+
+def main(directory, dep_path_mapping):
+    cargo_toml_files = find_cargo_toml_files(directory)
+    for cargo_toml_path in cargo_toml_files:
+        process_cargo_file(cargo_toml_path, dep_path_mapping, directory)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Modify dependencies in Cargo.toml files within a directory.")
+    parser.add_argument("directory", help="Directory to search for Cargo.toml files")
+    args = parser.parse_args()
+
+    # Define risc0, risc0-ethereum and bonsai dependencies and their paths here
+    # Add more dependencies and paths as needed
+    dep_path_mapping = {
+        "bonsai-rest-api-mock": "risc0/bonsai/rest-api-mock",
+        "bonsai-sdk": "risc0/bonsai/sdk",
+        "risc0-binfmt": "risc0/risc0/binfmt",
+        "risc0-build": "risc0/risc0/build",
+        "risc0-build-kernel": "risc0/risc0/build_kernel",
+        "risc0-circuit-recursion": "risc0/risc0/circuit/recursion",
+        "risc0-circuit-recursion-sys": "risc0/risc0/circuit/recursion-sys",
+        "risc0-circuit-rv32im": "risc0/risc0/circuit/rv32im",
+        "risc0-circuit-rv32im-sys": "risc0/risc0/circuit/rv32im-sys",
+        "risc0-core": "risc0/risc0/core",
+        "risc0-groth16": "risc0/risc0/groth16",
+        "risc0-r0vm": "risc0/risc0/r0vm",
+        "risc0-sys": "risc0/risc0/sys",
+        "risc0-zkp": "risc0/risc0/zkp",
+        "risc0-zkvm": "risc0/risc0/zkvm",
+        "risc0-zkvm-platform": "risc0/risc0/zkvm/platform",
+        "risc0-ethereum-contracts": "risc0-ethereum/contracts",
+        "risc0-build-ethereum": "risc0-ethereum/build",
+        "risc0-forge-ffi": "risc0-ethereum/ffi",
+        "risc0-ethereum-relay": "risc0-ethereum/relay",
+        "risc0-ethereum-relay-test-methods": "risc0-ethereum/relay/tests/methods",
+    }
+
+    main(os.path.normpath(args.directory), dep_path_mapping)

.github/workflows/bonsai.yml

@@ -53,56 +53,21 @@ jobs:
         uses: actions/checkout@v4
         with:
           lfs: true
-          submodules: recursive
 
       # Install dependencies.
       - uses: ./.github/actions/rustup
       - uses: ./.github/actions/sccache
         with:
           key: Linux-default
 
-      - name: Install Foundry
-        uses: risc0/foundry-toolchain@2fe7e70b520f62368a0e3c464f997df07ede420f
-        with:
-          version: nightly-09fe3e041369a816365a020f715ad6f94dbce9f2
-
       - name: Cargo install cargo-risczero
         run: cargo install --force --path risc0/cargo-risczero
 
       - run: cargo risczero install --version $RISC0_TOOLCHAIN_VERSION
 
-      # Lint
-      - name: check solidity code formatting in bonsai/ethereum
-        run: forge fmt --check
-        working-directory: bonsai/ethereum
-
-      - name: check solidity code formatting in bonsai/examples/governance
-        run: forge fmt --check
-        working-directory: bonsai/examples/governance
-
       # Test
       - name: run cargo tests in bonsai
         run: cargo test --locked -- --include-ignored --nocapture
         working-directory: bonsai
 
-      - name: run cargo tests in bonsai/examples/governance
-        run: cargo test --locked
-        working-directory: bonsai/examples/governance
-
-      - name: run forge tests in bonsai/ethereum
-        run: forge test -vvv
-        working-directory: bonsai/ethereum
-
-      - name: run forge tests without the zkVM in bonsai/examples/governance
-        env:
-          TEST_USE_ZKVM: false
-        run: forge test -vvv
-        working-directory: bonsai/examples/governance
-
-      - name: run forge tests with the zkvm in bonsai/examples/governance
-        env:
-          TEST_USE_ZKVM: true
-        run: forge test -vvv
-        working-directory: bonsai/examples/governance
-
       - run: sccache --show-stats

.github/workflows/main.yml

@@ -131,14 +131,12 @@ jobs:
       - run: cargo fmt --all --check
       - run: cargo fmt --all --check --manifest-path benchmarks/Cargo.toml
       - run: cargo fmt --all --check --manifest-path bonsai/Cargo.toml
-      - run: cargo fmt --all --check --manifest-path bonsai/examples/governance/Cargo.toml
       - run: cargo fmt --all --check --manifest-path examples/Cargo.toml
       - run: cargo fmt --all --check --manifest-path tools/crates-validator/Cargo.toml
       - run: cargo clippy -p risc0-zkvm
       - run: cargo sort --workspace --check
       - run: cargo sort --workspace --check benchmarks
       - run: cargo sort --workspace --check bonsai
-      - run: cargo sort --workspace --check bonsai/examples/governance
       - run: cargo sort --workspace --check examples
       - run: cargo sort --workspace --check tools/crates-validator
       - run: cargo rdme -c
@@ -189,15 +187,11 @@ jobs:
         run: |
           cargo test -F $FEATURE -F prove --workspace --timings --no-run \
             --exclude doc-test \
-            --exclude bonsai-ethereum-contracts \
-            --exclude bonsai-ethereum-relay \
             --exclude bonsai-rest-api-mock
       - name: test workspace
         run: |
           cargo test -F $FEATURE -F prove --workspace --timings \
             --exclude doc-test \
-            --exclude bonsai-ethereum-contracts \
-            --exclude bonsai-ethereum-relay \
             --exclude bonsai-rest-api-mock
       - uses: actions/upload-artifact@v3
         with:
@@ -213,8 +207,6 @@ jobs:
       - run: |
           cargo check -F $FEATURE --benches --workspace \
             --exclude doc-test \
-            --exclude bonsai-ethereum-contracts \
-            --exclude bonsai-ethereum-relay \
             --exclude bonsai-rest-api-mock
       - run: cargo check -p risc0-build
         if: matrix.device == 'cpu'
@@ -283,9 +275,6 @@ jobs:
       - run: "git checkout -f $(git -c user.name=x -c user.email=x@x commit-tree $(git hash-object -t tree /dev/null) < /dev/null) || :"
       - uses: actions/checkout@v4
       - uses: ./.github/actions/rustup
-      - uses: risc0/foundry-toolchain@2fe7e70b520f62368a0e3c464f997df07ede420f
-        with:
-          version: nightly-09fe3e041369a816365a020f715ad6f94dbce9f2
       - uses: ./.github/actions/sccache
         with:
           key: macOS-default

.github/workflows/risc0-ethereum.yml

@@ -0,0 +1,157 @@
+name: risc0-ethereum CI
+
+on:
+  merge_group:
+  pull_request:
+    branches: [main, "release-*"]
+  workflow_dispatch:
+    inputs:
+      risc0-ethereum-branch:
+        description: "risc0-ethereum branch to checkout"
+        required: false
+        default: 'main'
+      bonsai-foundry-template-branch:
+        description: "bonsai-foundry-template branch to checkout"
+        required: false
+        default: 'main'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+# this is needed to gain access via OIDC to the S3 bucket for caching
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  RISC0_TOOLCHAIN_VERSION: v2024-02-08.1
+  RISC0_PATH: ${{ github.workspace }}/risc0
+  RISC0_ETHEREUM_PATH: ${{ github.workspace }}/risc0-ethereum
+  BONSAI_FOUNDRY_TEMPLATE_PATH: ${{ github.workspace }}/bonsai-foundry-template
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read
+    outputs:
+      test: ${{ steps.filter.outputs.test }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: risc0/paths-filter@4067d885736b84de7c414f582ac45897079b0a78
+        id: filter
+        with:
+          filters: |
+            base: &base
+              - .github/actions/**
+              - .github/workflows/risc0-ethereum.yml
+              - Cargo.toml
+              - rust-toolchain.toml
+              - Cargo.lock
+            test:
+              - *base
+              - bonsai/**
+              - risc0/**
+
+  # see: https://github.com/orgs/community/discussions/26822
+  risc0-ethereum-status-check:
+    if: always()
+    needs:
+      - test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check all job status
+        # see https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
+        # see https://stackoverflow.com/a/67532120/4907315
+        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
+        run: exit 1
+
+  test:
+    if: ${{ needs.changes.outputs.test == 'true' || github.event_name == 'workflow_dispatch'}}
+    needs: changes
+    runs-on: [self-hosted, prod, Linux, cpu, docker]
+    env:
+      RUST_BACKTRACE: full
+    steps:
+      # This is a workaround from: https://github.com/actions/checkout/issues/590#issuecomment-970586842
+      - run: "git checkout -f $(git -c user.name=x -c user.email=x@x commit-tree $(git hash-object -t tree /dev/null) < /dev/null) || :"
+      # Checkout risc0
+      - uses: actions/checkout@v4
+        with:
+          path: 'risc0'
+      # Checkout risc0-ethereum
+      - name: Checkout risc0-ethereum
+        uses: actions/checkout@v4
+        with:
+          repository: 'risc0/risc0-ethereum'
+          token: ${{ secrets.GITHUB_TOKEN }}
+          submodules: recursive
+          path: 'risc0-ethereum'
+          ref: ${{ github.event.inputs.risc0-ethereum-branch || 'main' }}
+      # Checkout bonsai-foundry-template
+      - name: Checkout foundry-template
+        uses: actions/checkout@v4
+        with:
+          repository: 'risc0/bonsai-foundry-template'
+          token: ${{ secrets.GITHUB_TOKEN }}
+          submodules: recursive
+          path: 'bonsai-foundry-template'
+          ref: ${{ github.event.inputs.bonsai-foundry-template-branch || 'main' }}
+      - uses: ./risc0/.github/actions/rustup
+      - uses: ./risc0/.github/actions/sccache
+        with:
+          key: Linux-default
+      # Installing dev tooling
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Install python dependencies
+        run: python -m pip install --upgrade toml
+      - name: Install risczero toolchain
+        run: |
+          cargo install --force --path risc0/cargo-risczero
+          cargo risczero install --version $RISC0_TOOLCHAIN_VERSION
+        working-directory: risc0
+      - uses: risc0/foundry-toolchain@2fe7e70b520f62368a0e3c464f997df07ede420f
+        with:
+          version: nightly-09fe3e041369a816365a020f715ad6f94dbce9f2
+      # Apply local patch
+      - name: Local patch on risc0-ethereum
+        run: python ${{ env.RISC0_PATH }}/.github/cargo_local_patch.py  .
+        working-directory:  ${{ env.RISC0_ETHEREUM_PATH }}
+      - name: Local patch on bonsai-foundry-template
+        run: python ${{ env.RISC0_PATH }}/.github/cargo_local_patch.py  .
+        working-directory:  ${{ env.BONSAI_FOUNDRY_TEMPLATE_PATH }}
+      # Run Bootstrap-groth16 to update test receipt for checking
+      # the latest groth16 prover against the solidity verifier on risc0-ethereum
+      - name: Bootstrap-groth16 test receipt
+        run: cargo xtask bootstrap-groth16 --test-receipt-only
+        env:
+          RISC0_ETHEREUM_PATH: ${{ env.RISC0_ETHEREUM_PATH }}
+        working-directory: ${{ env.RISC0_PATH }}
+      # Build and test risc0-ethereum
+      - name: build risc0-ethereum
+        run: cargo test --workspace --no-run
+        working-directory: ${{ env.RISC0_ETHEREUM_PATH }}
+      - name: test risc0-ethereum
+        run: cargo test --workspace
+        working-directory: ${{ env.RISC0_ETHEREUM_PATH }}
+      - name: forge test on risc0-ethereum
+        run: forge test -vvv
+        working-directory: ${{ env.RISC0_ETHEREUM_PATH }}/contracts
+      # Build and test bonsai-foundry-template
+      - name: build bonsai-foundry-template
+        run: cargo build
+        working-directory: ${{ env.BONSAI_FOUNDRY_TEMPLATE_PATH }}
+      - name: build bonsai-foundry-template contracts
+        run: forge build
+        working-directory: ${{ env.BONSAI_FOUNDRY_TEMPLATE_PATH }}
+      - name: run foundry tests in dev mode
+        env:
+          RISC0_DEV_MODE: true
+        run: forge test -vvv
+        working-directory: ${{ env.BONSAI_FOUNDRY_TEMPLATE_PATH }}
+      - run: sccache --show-stats