devop test

.github/workflows/autogen.yml

@@ -5,29 +5,13 @@ on:
     types: [synchronize, opened, reopened, labeled]
 
 jobs:
-  wakeup:
-    if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'allow-autogen')
-    name: Wake up self-hosted runner
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@v2
-      - run: .github/workflows/wakeup_runner.sh
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
-          AWS_INSTANCE_ID: ${{ secrets.AWS_INSTANCE_ID }}
-
   gen:
+    if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'allow-autogen')
     timeout-minutes: 7200
     concurrency:
       group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.pull_request.number }}
       cancel-in-progress: true
-    env:
-      COMPOSE_PROJECT_NAME: ${{ github.run_id }}
-    needs: [wakeup]
-    runs-on: heavy-worker
+    runs-on: ["${{github.run_id}}", self-hosted, r6a.24xlarge]
     steps:
       - uses: actions/checkout@v2
         with:
@@ -39,7 +23,6 @@ jobs:
 
       - name: Build docker images
         run: |
-          docker compose down -v --remove-orphans || true
           docker compose build dev
 
       - name: Contracts

.github/workflows/test.yml → .github/workflows/coverage.yml

@@ -4,30 +4,15 @@ on:
   push:
     branches:
       - master
-  pull_request_target:
+  pull_request:
 
 jobs:
-  wakeup:
-    name: Wake up self-hosted runner
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@v2
-      - run: .github/workflows/wakeup_runner.sh
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
-          AWS_INSTANCE_ID: ${{ secrets.AWS_INSTANCE_ID }}
-
   main:
+    if: github.event.pull_request.draft == false
     concurrency:
       group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.pull_request.number }}
       cancel-in-progress: true
-    env:
-      COMPOSE_PROJECT_NAME: ${{ github.run_id }}
-    needs: [wakeup]
-    runs-on: heavy-worker
+    runs-on: ["${{github.run_id}}", self-hosted, r6a.2xlarge]
     steps:
       - uses: actions/checkout@v2
         with:

.github/workflows/github-ops.yml

@@ -0,0 +1,25 @@
+name: github-ops
+
+on:
+  workflow_dispatch:
+  workflow_run:
+    workflows: ["*"]
+    types:
+      - requested
+      - completed
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  ping:
+    runs-on: ubuntu-latest
+    timeout-minutes: 1
+    steps:
+      - name: Ping runner workflow
+        env:
+          # only needs public repo scope
+          GH_OPS_PAT: ${{ secrets.GH_OPS_PAT }}
+        run: |
+          curl -H "authorization: token $GH_OPS_PAT" 'https://api.github.com/repos/privacy-scaling-explorations/github-ops/dispatches' -d '{"event_type":"zkevm-chain"}'

.github/workflows/pi-eth-transfer.yml

@@ -0,0 +1,30 @@
+name: Pi Circuit aggregation eth_transfer
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [synchronize, opened, reopened, labeled]
+
+jobs:
+  test:
+    if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'ci-pi-eth-transfer')
+    timeout-minutes: 7200
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.pull_request.number }}
+      cancel-in-progress: true
+    runs-on: ["${{github.run_id}}", self-hosted, r6a.24xlarge]
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup
+        run: cp .env.example .env
+
+      - name: Build docker images
+        run: |
+          docker compose down -v --remove-orphans || true
+          docker compose build bootnode dev
+
+      - name: Pi Circuit aggregation eth_transfer
+        if: always()
+        run: |
+          docker compose -f docker-compose.yml -f docker-compose-perf.yml run --use-aliases --no-TTY --rm --entrypoint bash dev -c 'COORDINATOR_AGGREGATE_PROOF=true COORDINATOR_CIRCUIT_NAME=pi ./scripts/heavy_ci.sh eth_transfer'
+          ./scripts/ci_commit_errors.sh pi-ag-eth_transfer

.github/workflows/pi-native-withdraw.yml

@@ -0,0 +1,30 @@
+name: Pi Circuit aggregation native_withdraw
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [synchronize, opened, reopened, labeled]
+
+jobs:
+  test:
+    if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'ci-pi-native-withdraw')
+    timeout-minutes: 7200
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.pull_request.number }}
+      cancel-in-progress: true
+    runs-on: ["${{github.run_id}}", self-hosted, r6a.24xlarge]
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup
+        run: cp .env.example .env
+
+      - name: Build docker images
+        run: |
+          docker compose down -v --remove-orphans || true
+          docker compose build bootnode dev
+
+      - name: Pi Circuit aggregation native_withdraw
+        if: always()
+        run: |
+          docker compose -f docker-compose.yml -f docker-compose-perf.yml run --use-aliases --no-TTY --rm --entrypoint bash dev -c 'COORDINATOR_AGGREGATE_PROOF=true COORDINATOR_CIRCUIT_NAME=pi ./scripts/heavy_ci.sh native_withdraw'
+          ./scripts/ci_commit_errors.sh pi-ag-native_withdraw

.github/workflows/super-eth-transfer.yml

@@ -0,0 +1,30 @@
+name: Super Circuit aggregation eth_transfer
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [synchronize, opened, reopened, labeled]
+
+jobs:
+  test:
+    if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'ci-super-eth-transfer')
+    timeout-minutes: 7200
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.pull_request.number }}
+      cancel-in-progress: true
+    runs-on: ["${{github.run_id}}", self-hosted, r6a.24xlarge]
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup
+        run: cp .env.example .env
+
+      - name: Build docker images
+        run: |
+          docker compose down -v --remove-orphans || true
+          docker compose build bootnode dev
+
+      - name: Super Circuit aggregation eth_transfer
+        if: always()
+        run: |
+          docker compose -f docker-compose.yml -f docker-compose-perf.yml run --use-aliases --no-TTY --rm --entrypoint bash dev -c 'COORDINATOR_AGGREGATE_PROOF=true COORDINATOR_CIRCUIT_NAME=super ./scripts/heavy_ci.sh eth_transfer'
+          ./scripts/ci_commit_errors.sh super-ag-eth_transfer

.github/workflows/super-native-withdraw.yml

@@ -0,0 +1,30 @@
+name: Super Circuit aggregation native_withdraw
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [synchronize, opened, reopened, labeled]
+
+jobs:
+  test:
+    if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'ci-super-native-withdraw')
+    timeout-minutes: 7200
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.pull_request.number }}
+      cancel-in-progress: true
+    runs-on: ["${{github.run_id}}", self-hosted, r6a.48xlarge]
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup
+        run: cp .env.example .env
+
+      - name: Build docker images
+        run: |
+          docker compose down -v --remove-orphans || true
+          docker compose build bootnode dev
+
+      - name: Super Circuit aggregation native_withdraw
+        if: always()
+        run: |
+          docker compose -f docker-compose.yml -f docker-compose-perf.yml run --use-aliases --no-TTY --rm --entrypoint bash dev -c 'COORDINATOR_AGGREGATE_PROOF=true COORDINATOR_CIRCUIT_NAME=super ./scripts/heavy_ci.sh native_withdraw'
+          ./scripts/ci_commit_errors.sh super-ag-native_withdraw

coordinator/src/utils.rs

@@ -13,7 +13,7 @@ use hyper::client::HttpConnector;
 use hyper::Uri;
 use zkevm_common::json_rpc::jsonrpc_request_client;
 
-pub const RPC_REQUEST_TIMEOUT: u64 = 15000;
+pub const RPC_REQUEST_TIMEOUT: u64 = 30000;
 
 /// may override any pending transactions
 pub async fn send_transaction_to_l1(

scripts/heavy_ci.sh

@@ -11,8 +11,9 @@ trap 'pkill --parent $$' TERM EXIT INT
 
 cargo build --release --bin prover_rpcd
 env time --output PROVER_STATS.txt --verbose -- \
-  perf stat \
-  -o PROVER_PERF.txt \
+  cargo run --release --bin prover_rpcd 2>&1 | xz > PROVER_LOG.txt.xz &
+PID=$!
+perf stat --pid $PID -I 300000 -o PROVER_PERF.txt \
   -e stalled-cycles-backend \
   -e stalled-cycles-frontend \
   -e instructions \
@@ -30,10 +31,7 @@ env time --output PROVER_STATS.txt --verbose -- \
   -e l2_cache_accesses_from_ic_misses \
   -e ic_tag_hit_miss.all_instruction_cache_accesses \
   -e ic_tag_hit_miss.instruction_cache_hit \
-  -e ic_tag_hit_miss.instruction_cache_miss \
-  -- \
-  cargo run --release --bin prover_rpcd 2>&1 | xz > PROVER_LOG.txt.xz &
-PID=$!
+  -e ic_tag_hit_miss.instruction_cache_miss &
 
 # sleep a bit in case the geth nodes are not up yet
 sleep 3
@@ -55,6 +53,11 @@ if [ $status -eq 0 ]; then
   exit 0
 fi
 
+# if there are not failed proof requests, then something else failed
+if [ "${FAILED_BLOCKS}" = "" ]; then
+  exit 1
+fi
+
 # error collection
 mkdir errors