perf: reduce memory consumption of witness generation (#696)

## What ❔

Uses an [updated
zkevm_test_harness](matter-labs/era-zkevm_test_harness#57)
that uses less memory. The updated crate takes callback functions
instead of returning a vector of all results.

We may have to think about how to not keep the database open the whole
time the callbacks are coming in.

## Why ❔

Proving batches currently uses memory proportional to the batch size,
which is not ok for large batches.

---------

Co-authored-by: Fedor Sakharov <fedor.sakharov@gmail.com>
Co-authored-by: AntonD3 <74021421+AntonD3@users.noreply.github.com>
Co-authored-by: Stanislav Breadless <stanislavbezkor@gmail.com>
Co-authored-by: perekopskiy <53865202+perekopskiy@users.noreply.github.com>
Co-authored-by: pompon0 <pompon.pompon@gmail.com>
Co-authored-by: Dustin Brickwood <dustinbrickwood204@gmail.com>
Co-authored-by: Igor Borodin <hatemosphere@protonmail.com>

prover/proof_fri_compressor/Cargo.toml

@@ -21,7 +21,7 @@ vk_setup_data_generator_server_fri = { path = "../vk_setup_data_generator_server
 vlog = { path = "../../core/lib/vlog" }
 
 zkevm_test_harness_1_3_3 = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "v1.3.3", package = "zkevm_test_harness" }
-zkevm_test_harness = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "v1.4.1" }
+zkevm_test_harness = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "jms-fix-memory-use" }
 
 anyhow = "1.0"
 tracing = "0.1"

prover/prover_fri/Cargo.toml

@@ -20,13 +20,13 @@ zksync_prover_fri_utils = { path = "../prover_fri_utils" }
 zksync_prover_fri_types = { path = "../prover_fri_types" }
 zksync_utils = { path = "../../core/lib/utils" }
 vk_setup_data_generator_server_fri = { path = "../vk_setup_data_generator_server_fri" }
-shivini = { git = "https://github.com/matter-labs/era-shivini.git", branch = "v1.4.1", optional = true, features = [
+shivini = { git = "https://github.com/matter-labs/era-shivini.git", branch = "jms-fix-memory-use", optional = true, features = [
     "circuit_definitions",
     "zksync",
 ] }
 
-zkevm_test_harness = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "v1.4.1" }
-circuit_definitions = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "v1.4.1", features = [
+zkevm_test_harness = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "jms-fix-memory-use" }
+circuit_definitions = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "jms-fix-memory-use", features = [
     "log_tracing",
 ] }
 

prover/prover_fri_types/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2021"
 zksync_object_store = { path = "../../core/lib/object_store" }
 zksync_types = { path = "../../core/lib/types" }
 
-circuit_definitions = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "v1.4.1", features = [
+circuit_definitions = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "jms-fix-memory-use", features = [
     "log_tracing",
 ] }
 

prover/vk_setup_data_generator_server_fri/Cargo.toml

@@ -26,11 +26,11 @@ zksync_types = { path = "../../core/lib/types" }
 zksync_prover_fri_types = { path = "../prover_fri_types" }
 
 zkevm_test_harness_1_3_3 = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "v1.3.3", package = "zkevm_test_harness" }
-zkevm_test_harness = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "v1.4.1" }
-circuit_definitions = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "v1.4.1", features = [
+zkevm_test_harness = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "jms-fix-memory-use" }
+circuit_definitions = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "jms-fix-memory-use", features = [
     "log_tracing",
 ] }
-shivini = { git = "https://github.com/matter-labs/era-shivini.git", branch = "v1.4.1", optional = true }
+shivini = { git = "https://github.com/matter-labs/era-shivini.git", branch = "jms-fix-memory-use", optional = true }
 zksync_config = { path = "../../core/lib/config" }
 zksync_env_config = { path = "../../core/lib/env_config" }