ERROR zkevm_circuits::witness::mpt - roots non consistent

[Gao-Chuan]

Hi Scroll team,

An error raised when I was testing scroll-zkevm. It looks like an internal error. Could you please take a look?

My code and the errors are pasted below:

The error:

2023-06-11T00:42:05.556480238+00:00 ERROR zkevm_circuits::witness::mpt - roots non consistent (0x4c24b736c6ad0723c18abc19427cdc3f04c587144ace83fa8f4b4fdc235854a,0x14d0144696eef8219c87d939389abfba43b018cf6dc63e28b6b17505ef54594f) vs (0x4c24b736c6ad0723c18abc19427cdc3f04c587144ace83fa8f4b4fdc235854a,0x28595a20069852e700e8fd2554d136c1cb3e087a60018a27b719c88b6ad8f6eb)

Note: there's a segmentation fault in the end. prover.create_target_circuit_proof_batch() does not return.

The log file:

2023-06-10T21:00:51.201968704+00:00 INFO zkevm::utils - git version prealpha-v2.0-25-g1f24911-modified
2023-06-10T21:00:51.202490876+00:00 INFO zkevm::utils - load_or_create_params /home/ubuntu/scroll-zkevm/param/params25
2023-06-10T21:00:51.202604204+00:00 INFO zkevm::utils - start creating params with degree 25
2023-06-11T00:06:03.628418193+00:00 INFO zkevm::utils - create params successfully!
2023-06-11T00:37:32.127070146+00:00 INFO zkevm::prover::util - loaded parameters for degrees 20 and 25
2023-06-11T00:37:32.127300581+00:00 INFO zkevm::prover::util - params g2 (Fq2 { c0: 0x1800deef121f1e76426a00665e5c4479674322d4f75edadd46debd5cd992f6ed, c1: 0x198e9393920d483a7260bfb731fb5d25f1aa493335a9e71297e485b7aef312c2 }, Fq2 { c0: 0x12c85ea5db8c6deb4aab71808dcb408fe3d1e7690c43d37b4ce6cc0166fa7daa, c1: 0x090689d0585ff075ec9e99ad690c3395bc4b313370b38ef355acdadcd122975b }) s_g2 (Fq2 { c0: 0x097aa085249a574ff7d1aaa2b498623f2923d159833cfde765a657aff8e0b527, c1: 0x2c358c680be102b15427cd4f58158ea3267a938ac04e1de564fd577bb7258960 }, Fq2 { c0: 0x1899775af20ee6b6b62d0b1c9dd6ad2e954af70764a84584c7cec5e522339fd9, c1: 0x0f77730977b52fe9c4d6cee4c281a3ace737795a9788d0ac3d9f3a9692708c75 })
2023-06-11T00:37:32.146821161+00:00 INFO zkevm::circuit::builder - check capacity of block traces, num_block 1, num_tx 1, tx total len 68
2023-06-11T00:37:38.172389249+00:00 INFO bus_mapping::circuit_input_builder - handling block Some(5), tx num 1
2023-06-11T00:37:38.172574898+00:00 INFO bus_mapping::circuit_input_builder - handling 0th tx(inner idx: 0): 0x1803da2b53edba96f1122f6a1c7edb58b47489956f4f5e96028369f71b30a91d rwc 1, to: Some(0x03f8133dd5ed58838b20af1296f62f44e69baa48), input_len 68
2023-06-11T00:37:38.177600653+00:00 WARN bus_mapping::evm::opcodes - geth error OutOfGas(SloadSstore) occurred in  SSTORE at pc 0x000b16
2023-06-11T00:37:38.178022338+00:00 INFO bus_mapping::circuit_input_builder - handling block done, total gas 49854
2023-06-11T00:37:46.652621750+00:00 ERROR zkevm_circuits::witness::mpt - roots non consistent (0x4c24b736c6ad0723c18abc19427cdc3f04c587144ace83fa8f4b4fdc235854a,0x14d0144696eef8219c87d939389abfba43b018cf6dc63e28b6b17505ef54594f) vs (0x4c24b736c6ad0723c18abc19427cdc3f04c587144ace83fa8f4b4fdc235854a,0x28595a20069852e700e8fd2554d136c1cb3e087a60018a27b719c88b6ad8f6eb)
2023-06-11T00:37:46.652851744+00:00 INFO mpt_zktrie::state::witness - account data {
    0x5300000000000000000000000000000000000002: AccountData {
        nonce: 0,
        balance: 0,
        keccak_code_hash: 0x0fabb5b0f58ec2922e2969f4dadb6d1395b49ecd40feff93e01212ae848355d4,
        poseidon_code_hash: 0x10e77cae1c507f967948c6cd114e74ed65f662e365c7d6993e97f78ce8982528,
        code_size: 2164,
        storage_root: 0x256484a85a716c24ed051180b7f705d3f8a890641a8df27c18d0e6ec3a024473,
    },
    0x03f8133dd5ed58838b20af1296f62f44e69baa48: AccountData {
        nonce: 1,
        balance: 0,
        keccak_code_hash: 0x16fc66d15010e6213d2a009f57ed8e847717ea0b83eeb37cd322e9ad1b018a3e,
        poseidon_code_hash: 0x0d85b09a93d5ed99a87d27dcf6d50e4459d16bb694e70f89eefcb745ea1c85e7,
        code_size: 4507,
        storage_root: 0x1b725ef9a53b6069ef6daeb73a063cf042a7c995dc1771f3908f0d7ce19f32a6,
    },
    0x1c5a77d9fa7ef466951b2f01f724bca3a5820b63: AccountData {
        nonce: 5,
        balance: 904625697166532776746648320380374280103671755200316706558261057343822587536,
        keccak_code_hash: 0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470,
        poseidon_code_hash: 0x2098f5fb9e239eab3ceac3f27b81e481dc3124d55ffed523a839ee8446b64864,
        code_size: 0,
        storage_root: 0x0000000000000000000000000000000000000000000000000000000000000000,
    },
    0x5300000000000000000000000000000000000005: AccountData {
        nonce: 0,
        balance: 1317718015514992,
        keccak_code_hash: 0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470,
        poseidon_code_hash: 0x2098f5fb9e239eab3ceac3f27b81e481dc3124d55ffed523a839ee8446b64864,
        code_size: 0,
        storage_root: 0x0000000000000000000000000000000000000000000000000000000000000000,
    },
    0x5300000000000000000000000000000000000000: AccountData {
        nonce: 0,
        balance: 0,
        keccak_code_hash: 0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470,
        poseidon_code_hash: 0x2098f5fb9e239eab3ceac3f27b81e481dc3124d55ffed523a839ee8446b64864,
        code_size: 0,
        storage_root: 0x0000000000000000000000000000000000000000000000000000000000000000,
    },
}
2023-06-11T00:41:57.060325708+00:00 INFO bus_mapping::circuit_input_builder - handling block Some(5), tx num 1
2023-06-11T00:41:57.060533364+00:00 INFO bus_mapping::circuit_input_builder - handling 0th tx(inner idx: 0): 0x1803da2b53edba96f1122f6a1c7edb58b47489956f4f5e96028369f71b30a91d rwc 1, to: Some(0x03f8133dd5ed58838b20af1296f62f44e69baa48), input_len 68
2023-06-11T00:41:57.064486391+00:00 WARN bus_mapping::evm::opcodes - geth error OutOfGas(SloadSstore) occurred in  SSTORE at pc 0x000b16
2023-06-11T00:41:57.064757045+00:00 INFO bus_mapping::circuit_input_builder - handling block done, total gas 49854
2023-06-11T00:42:05.556480238+00:00 ERROR zkevm_circuits::witness::mpt - roots non consistent (0x4c24b736c6ad0723c18abc19427cdc3f04c587144ace83fa8f4b4fdc235854a,0x14d0144696eef8219c87d939389abfba43b018cf6dc63e28b6b17505ef54594f) vs (0x4c24b736c6ad0723c18abc19427cdc3f04c587144ace83fa8f4b4fdc235854a,0x28595a20069852e700e8fd2554d136c1cb3e087a60018a27b719c88b6ad8f6eb)
2023-06-11T00:42:05.556712654+00:00 INFO mpt_zktrie::state::witness - account data {
    0x5300000000000000000000000000000000000000: AccountData {
        nonce: 0,
        balance: 0,
        keccak_code_hash: 0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470,
        poseidon_code_hash: 0x2098f5fb9e239eab3ceac3f27b81e481dc3124d55ffed523a839ee8446b64864,
        code_size: 0,
        storage_root: 0x0000000000000000000000000000000000000000000000000000000000000000,
    },
    0x5300000000000000000000000000000000000005: AccountData {
        nonce: 0,
        balance: 1317718015514992,
        keccak_code_hash: 0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470,
        poseidon_code_hash: 0x2098f5fb9e239eab3ceac3f27b81e481dc3124d55ffed523a839ee8446b64864,
        code_size: 0,
        storage_root: 0x0000000000000000000000000000000000000000000000000000000000000000,
    },
    0x03f8133dd5ed58838b20af1296f62f44e69baa48: AccountData {
        nonce: 1,
        balance: 0,
        keccak_code_hash: 0x16fc66d15010e6213d2a009f57ed8e847717ea0b83eeb37cd322e9ad1b018a3e,
        poseidon_code_hash: 0x0d85b09a93d5ed99a87d27dcf6d50e4459d16bb694e70f89eefcb745ea1c85e7,
        code_size: 4507,
        storage_root: 0x1b725ef9a53b6069ef6daeb73a063cf042a7c995dc1771f3908f0d7ce19f32a6,
    },
    0x5300000000000000000000000000000000000002: AccountData {
        nonce: 0,
        balance: 0,
        keccak_code_hash: 0x0fabb5b0f58ec2922e2969f4dadb6d1395b49ecd40feff93e01212ae848355d4,
        poseidon_code_hash: 0x10e77cae1c507f967948c6cd114e74ed65f662e365c7d6993e97f78ce8982528,
        code_size: 2164,
        storage_root: 0x256484a85a716c24ed051180b7f705d3f8a890641a8df27c18d0e6ec3a024473,
    },
    0x1c5a77d9fa7ef466951b2f01f724bca3a5820b63: AccountData {
        nonce: 5,
        balance: 904625697166532776746648320380374280103671755200316706558261057343822587536,
        keccak_code_hash: 0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470,
        poseidon_code_hash: 0x2098f5fb9e239eab3ceac3f27b81e481dc3124d55ffed523a839ee8446b64864,
        code_size: 0,
        storage_root: 0x0000000000000000000000000000000000000000000000000000000000000000,
    },
}
2023-06-11T00:42:05.560718561+00:00 INFO zkevm::prover::inner_circuit - proving batch of len 1, batch metric BatchMetric { num_block: 1, num_tx: 1, num_step: 298 }
2023-06-11T00:45:41.431136025+00:00 INFO zkevm_circuits::tx_circuit - TxCircuit::new(max_txs = 32, max_calldata = 400000, chain_id = 53077)
2023-06-11T00:45:49.985448538+00:00 INFO zkevm::prover::inner_circuit - Create super proof of block 0x9adb…b929 ... block 0x9adb…b929, batch len 1
2023-06-11T00:45:51.993973205+00:00 INFO zkevm_circuits::tx_circuit - TxCircuit::new(max_txs = 32, max_calldata = 400000, chain_id = 534353)
2023-06-11T00:45:53.178631865+00:00 INFO zkevm_circuits::tx_circuit - after value_is_zero, meta.degree: 9
2023-06-11T00:45:53.656106861+00:00 INFO zkevm_circuits::tx_circuit - after end, meta.degree: 9
2023-06-11T00:49:10.506689326+00:00 INFO zkevm_circuits::keccak_circuit::keccak_packed_multi - multi_keccak assign with capacity: Some(1744)

My code:
demo --trace /home/ubuntu/scroll-zkevm/zkevm/tests/traces/erc20/erc20_1_transfer.json

use clap::Parser;
use log::info;
use std::collections::HashMap;
use std::fs;
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
use std::time::Instant;
use std::env;
use zkevm::{
    circuit::{SuperCircuit, AGG_DEGREE},
    prover::Prover,
    utils::{get_block_trace_from_file, init_env_and_log, load_or_create_params},
};

#[derive(Parser, Debug)]
#[clap(author, version, about, long_about = None)]
struct Args {
    /// Get params and write into file.
    #[clap(short, long = "params")]
    params_path: Option<String>,
    /// Get BlockTrace from file or dir.
    #[clap(short, long = "trace")]
    trace_path: Option<String>,
}

fn main() {
    let args = Args::parse();
    env::set_var("RUST_BACKTRACE", "1");
    let param_path = "/home/ubuntu/scroll-zkevm/param";
    init_env_and_log("prove");

    //
    // 1. instantiation the parameters and the prover
    //

    println!("Started!");

    let args = Args::parse();
    let agg_params = load_or_create_params(param_path, *AGG_DEGREE)
        .expect("failed to load or create params");

    let mut prover = Prover::from_params(agg_params);
    
    println!("Prover built");

    //
    // 2. read inner circuit proofs (a.k.a. SNARKs) from previous dumped file or
    //    convert block traces into proofs
    //

    // 2.1. Read traces from a directory or a file.
    let mut path_traces = HashMap::new();
    let trace_path = PathBuf::from(&args.trace_path.unwrap());
    if trace_path.is_dir() {
        for entry in fs::read_dir(trace_path).unwrap() {
            let path = entry.unwrap().path();
            if path.is_file() && path.to_str().unwrap().ends_with(".json") {
                let block_trace = get_block_trace_from_file(path.to_str().unwrap());
                path_traces.insert(path.file_stem().unwrap().to_os_string(), block_trace);
            }
        }
    } else {
        let block_trace = get_block_trace_from_file(trace_path.to_str().unwrap());
        path_traces.insert(trace_path.file_stem().unwrap().to_os_string(), block_trace);
    }
    let traces = path_traces.values().cloned().collect::<Vec<types::eth::BlockTrace>>();

    println!("Traces readen");

    // 2.2. Generate super circuit proof and dump it to output folder.
    let outer_now = Instant::now();
    let inner_proof_file_path = "/home/ubuntu/scroll-zkevm/output/super_proof.json";
    let proof = prover
                .create_target_circuit_proof_batch::<SuperCircuit>(&traces)
                .expect("Cannot generate evm_proof");

    println!("Proof generated!");

    // Dump inner circuit proof.
    proof.dump_to_file(&inner_proof_file_path)
        .expect("Cannot dump evm_proof");

    println!("finish generating all, elapsed: {:?}", outer_now.elapsed());
}