feat: Manual ClientIVC breakdown (#4778)

Add timers to more functions used in client IVC scheme. Add scripts to get benchmarks and analyze them, giving a coarse profile of client IVC execution in x86. Small cleanup: rename ivc_bench module to client_ivc_bench for uniformity and because we will have other IVC schemes. Results on commit c2ab55e ``` Benchmarking lock created at ~/BENCHMARK_IN_PROGRESS. client_ivc_bench 100% 16MB 49.3MB/s 00:00 2024-02-27T14:44:25+00:00 Running ./client_ivc_bench Run on (16 X 3597.59 MHz CPU s) CPU Caches: L1 Data 32 KiB (x8) L1 Instruction 32 KiB (x8) L2 Unified 1024 KiB (x8) L3 Unified 36608 KiB (x1) Load Average: 0.00, 0.00, 0.00 -------------------------------------------------------------------------------- Benchmark Time CPU Iterations UserCounters... -------------------------------------------------------------------------------- ClientIVCBench/Full/6 39503 ms 37113 ms 1 Decider::construct_proof=1 Decider::construct_proof(t)=759.097M ECCVMComposer::create_prover=1 ECCVMComposer::create_prover(t)=3.77773G ECCVMProver::construct_proof=1 ECCVMProver::construct_proof(t)=1.76856G Goblin::merge=12 Goblin::merge(t)=153.269M GoblinTranslatorCircuitBuilder::constructor=1 GoblinTranslatorCircuitBuilder::constructor(t)=61.9766M GoblinTranslatorComposer::create_prover=1 GoblinTranslatorComposer::create_prover(t)=114.006M GoblinTranslatorProver::construct_proof=1 GoblinTranslatorProver::construct_proof(t)=939.454M ProtogalaxyProver::fold_instances=11 ProtogalaxyProver::fold_instances(t)=18.0888G UltraComposer::create_prover_instance=12 UltraComposer::create_prover_instance(t)=6.31421G batch_mul_with_endomorphism=30 batch_mul_with_endomorphism(t)=563.904M commit=804 commit(t)=8.10774G compute_combiner=11 compute_combiner(t)=7.92258G compute_perturbator=10 compute_perturbator(t)=1.52886G construct_mock_folding_kernel=5 construct_mock_folding_kernel(t)=2.33398G construct_mock_function_circuit=7 construct_mock_function_circuit(t)=5.16551G Benchmarking lock deleted. client_ivc_bench.json 100% 3063 85.5KB/s 00:00 label ms % total construct_mock_function_circuit(t) 5166 13.08% construct_mock_folding_kernel(t) 2334 5.91% UltraComposer::create_prover_instance(t) 6314 15.98% ProtogalaxyProver::fold_instances(t) 18089 45.79% Decider::construct_proof(t) 759 1.92% ECCVMComposer::create_prover(t) 3778 9.56% GoblinTranslatorComposer::create_prover(t) 114 0.29% ECCVMProver::construct_proof(t) 1769 4.48% GoblinTranslatorProver::construct_proof(t) 939 2.38% Goblin::merge(t) 153 0.39% ```
AztecProtocol · Feb 27, 2024 · b4cfc89 · b4cfc89
1 parent 61067a5
commit b4cfc89
Show file tree

Hide file tree

Showing 16 changed files with 98 additions and 17 deletions.
diff --git a/barretenberg/cpp/scripts/analyze_client_ivc_bench.py b/barretenberg/cpp/scripts/analyze_client_ivc_bench.py
@@ -0,0 +1,44 @@
+import json
+from pathlib import Path
+
+PREFIX = Path("build-op-count-time")
+IVC_BENCH_JSON = Path("client_ivc_bench.json")
+BENCHMARK = "ClientIVCBench/Full/6"
+
+# Single out an independent set of functions accounting for most of BENCHMARK's real_time
+to_keep = [
+    "construct_mock_function_circuit(t)",
+    "construct_mock_folding_kernel(t)",
+    "UltraComposer::create_prover_instance(t)",
+    "ProtogalaxyProver::fold_instances(t)",
+    "Decider::construct_proof(t)",
+    "ECCVMComposer::create_prover(t)",
+    "GoblinTranslatorComposer::create_prover(t)",
+    "ECCVMProver::construct_proof(t)",
+    "GoblinTranslatorProver::construct_proof(t)",
+    "Goblin::merge(t)"
+]
+with open(PREFIX/IVC_BENCH_JSON, "r") as read_file:
+    read_result = json.load(read_file)
+    for _bench in read_result["benchmarks"]:
+        if _bench["name"] == BENCHMARK:
+            bench = _bench
+bench_components = dict(filter(lambda x: x[0] in to_keep, bench.items()))
+
+# For each kept time, get the proportion over all kept times.
+sum_of_kept_times_ms = sum(float(time)
+                           for _, time in bench_components.items())/1e6
+MAX_LABEL_LENGTH = max(len(label) for label in to_keep)
+column = {"function": "function", "ms": "ms", "%": "% sum"}
+print(
+    f"{column['function']:<{MAX_LABEL_LENGTH}}{column['ms']:>8}  {column['%']:>8}")
+for key in to_keep:
+    time_ms = bench[key]/1e6
+    print(f"{key:<{MAX_LABEL_LENGTH}}{time_ms:>8.0f}  {time_ms/sum_of_kept_times_ms:>8.2%}")
+
+# Validate that kept times account for most of the total measured time.
+total_time_ms = bench["real_time"]
+totals = '\nTotal time accounted for: {:.0f}ms/{:.0f}ms = {:.2%}'
+totals = totals.format(
+    sum_of_kept_times_ms, total_time_ms, sum_of_kept_times_ms/total_time_ms)
+print(totals)
diff --git a/barretenberg/cpp/scripts/benchmark_client_ivc.sh b/barretenberg/cpp/scripts/benchmark_client_ivc.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -eu
+
+TARGET="client_ivc_bench"
+FILTER="ClientIVCBench/Full/6$"
+BUILD_DIR=build-op-count-time
+
+# Move above script dir.
+cd $(dirname $0)/..
+
+# Measure the benchmarks with ops time counting
+./scripts/benchmark_remote.sh client_ivc_bench\
+                              "./client_ivc_bench --benchmark_filter=$FILTER\
+                                                  --benchmark_out=$TARGET.json\
+                                                  --benchmark_out_format=json"\
+                              op-count-time\
+                              build-op-count-time
+
+# Retrieve output from benching instance
+cd $BUILD_DIR
+scp $BB_SSH_KEY $BB_SSH_INSTANCE:$BB_SSH_CPP_PATH/build/$TARGET.json .
+
+# Analyze the results
+cd $(dirname $0)/..
+python3 ./scripts/analyze_client_ivc_bench.py
diff --git a/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt
@@ -2,7 +2,7 @@ add_subdirectory(basics_bench)
 add_subdirectory(decrypt_bench)
 add_subdirectory(goblin_bench)
 add_subdirectory(ipa_bench)
-add_subdirectory(ivc_bench)
+add_subdirectory(client_ivc_bench)
 add_subdirectory(pippenger_bench)
 add_subdirectory(plonk_bench)
 add_subdirectory(protogalaxy_bench)

diff --git a/barretenberg/cpp/src/barretenberg/benchmark/client_ivc_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/client_ivc_bench/CMakeLists.txt
@@ -0,0 +1 @@
+barretenberg_module(client_ivc_bench client_ivc stdlib_recursion stdlib_sha256 crypto_merkle_tree stdlib_primitives)
diff --git a/...tenberg/benchmark/ivc_bench/ivc.bench.cpp → ...ark/client_ivc_bench/client_ivc.bench.cpp b/...tenberg/benchmark/ivc_bench/ivc.bench.cpp → ...ark/client_ivc_bench/client_ivc.bench.cpp
@@ -1,10 +1,8 @@
 
 #include <benchmark/benchmark.h>
 
-#include "barretenberg/benchmark/ultra_bench/mock_proofs.hpp"
 #include "barretenberg/client_ivc/client_ivc.hpp"
 #include "barretenberg/common/op_count_google_bench.hpp"
-#include "barretenberg/goblin/goblin.hpp"
 #include "barretenberg/goblin/mock_circuits.hpp"
 #include "barretenberg/proof_system/circuit_builder/ultra_circuit_builder.hpp"
 #include "barretenberg/ultra_honk/ultra_composer.hpp"
@@ -18,7 +16,7 @@ namespace {
  * @brief Benchmark suite for the aztec client PG-Goblin IVC scheme
  *
  */
-class IvcBench : public benchmark::Fixture {
+class ClientIVCBench : public benchmark::Fixture {
   public:
     using Builder = GoblinUltraCircuitBuilder;
 
@@ -74,7 +72,7 @@ class IvcBench : public benchmark::Fixture {
  * @brief Benchmark the prover work for the full PG-Goblin IVC protocol
  *
  */
-BENCHMARK_DEFINE_F(IvcBench, Full)(benchmark::State& state)
+BENCHMARK_DEFINE_F(ClientIVCBench, Full)(benchmark::State& state)
 {
     ClientIVC ivc;
 
@@ -92,7 +90,7 @@ BENCHMARK_DEFINE_F(IvcBench, Full)(benchmark::State& state)
  * @brief Benchmark only the accumulation rounds
  *
  */
-BENCHMARK_DEFINE_F(IvcBench, Accumulate)(benchmark::State& state)
+BENCHMARK_DEFINE_F(ClientIVCBench, Accumulate)(benchmark::State& state)
 {
     ClientIVC ivc;
 
@@ -107,7 +105,7 @@ BENCHMARK_DEFINE_F(IvcBench, Accumulate)(benchmark::State& state)
  * @brief Benchmark only the Decider component
  *
  */
-BENCHMARK_DEFINE_F(IvcBench, Decide)(benchmark::State& state)
+BENCHMARK_DEFINE_F(ClientIVCBench, Decide)(benchmark::State& state)
 {
     ClientIVC ivc;
 
@@ -125,7 +123,7 @@ BENCHMARK_DEFINE_F(IvcBench, Decide)(benchmark::State& state)
  * @brief Benchmark only the ECCVM component
  *
  */
-BENCHMARK_DEFINE_F(IvcBench, ECCVM)(benchmark::State& state)
+BENCHMARK_DEFINE_F(ClientIVCBench, ECCVM)(benchmark::State& state)
 {
     ClientIVC ivc;
 
@@ -143,7 +141,7 @@ BENCHMARK_DEFINE_F(IvcBench, ECCVM)(benchmark::State& state)
  * @brief Benchmark only the Translator component
  *
  */
-BENCHMARK_DEFINE_F(IvcBench, Translator)(benchmark::State& state)
+BENCHMARK_DEFINE_F(ClientIVCBench, Translator)(benchmark::State& state)
 {
     ClientIVC ivc;
 
@@ -159,7 +157,7 @@ BENCHMARK_DEFINE_F(IvcBench, Translator)(benchmark::State& state)
 }
 
 #define ARGS                                                                                                           \
-    Arg(IvcBench::NUM_ITERATIONS_MEDIUM_COMPLEXITY)                                                                    \
+    Arg(ClientIVCBench::NUM_ITERATIONS_MEDIUM_COMPLEXITY)                                                              \
         ->Arg(1 << 0)                                                                                                  \
         ->Arg(1 << 1)                                                                                                  \
         ->Arg(1 << 2)                                                                                                  \
@@ -168,11 +166,11 @@ BENCHMARK_DEFINE_F(IvcBench, Translator)(benchmark::State& state)
         ->Arg(1 << 5)                                                                                                  \
         ->Arg(1 << 6)
 
-BENCHMARK_REGISTER_F(IvcBench, Full)->Unit(benchmark::kMillisecond)->ARGS;
-BENCHMARK_REGISTER_F(IvcBench, Accumulate)->Unit(benchmark::kMillisecond)->ARGS;
-BENCHMARK_REGISTER_F(IvcBench, Decide)->Unit(benchmark::kMillisecond)->ARGS;
-BENCHMARK_REGISTER_F(IvcBench, ECCVM)->Unit(benchmark::kMillisecond)->ARGS;
-BENCHMARK_REGISTER_F(IvcBench, Translator)->Unit(benchmark::kMillisecond)->ARGS;
+BENCHMARK_REGISTER_F(ClientIVCBench, Full)->Unit(benchmark::kMillisecond)->ARGS;
+BENCHMARK_REGISTER_F(ClientIVCBench, Accumulate)->Unit(benchmark::kMillisecond)->ARGS;
+BENCHMARK_REGISTER_F(ClientIVCBench, Decide)->Unit(benchmark::kMillisecond)->ARGS;
+BENCHMARK_REGISTER_F(ClientIVCBench, ECCVM)->Unit(benchmark::kMillisecond)->ARGS;
+BENCHMARK_REGISTER_F(ClientIVCBench, Translator)->Unit(benchmark::kMillisecond)->ARGS;
 
 } // namespace
 

diff --git a/barretenberg/cpp/src/barretenberg/benchmark/ivc_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/ivc_bench/CMakeLists.txt
diff --git a/barretenberg/cpp/src/barretenberg/eccvm/eccvm_composer.cpp b/barretenberg/cpp/src/barretenberg/eccvm/eccvm_composer.cpp
@@ -30,6 +30,8 @@ template <IsECCVMFlavor Flavor>
 ECCVMProver_<Flavor> ECCVMComposer_<Flavor>::create_prover(CircuitConstructor& circuit_constructor,
                                                            const std::shared_ptr<Transcript>& transcript)
 {
+    BB_OP_COUNT_TIME_NAME("ECCVMComposer::create_prover");
+
     compute_proving_key(circuit_constructor);
     compute_witness(circuit_constructor);
     compute_commitment_key(proving_key->circuit_size);

diff --git a/barretenberg/cpp/src/barretenberg/eccvm/eccvm_prover.cpp b/barretenberg/cpp/src/barretenberg/eccvm/eccvm_prover.cpp
@@ -282,6 +282,8 @@ template <IsECCVMFlavor Flavor> HonkProof& ECCVMProver_<Flavor>::export_proof()
 
 template <IsECCVMFlavor Flavor> HonkProof& ECCVMProver_<Flavor>::construct_proof()
 {
+    BB_OP_COUNT_TIME_NAME("ECCVMProver::construct_proof");
+
     execute_preamble_round();
 
     execute_wire_commitments_round();

diff --git a/barretenberg/cpp/src/barretenberg/goblin/goblin.hpp b/barretenberg/cpp/src/barretenberg/goblin/goblin.hpp
@@ -129,6 +129,7 @@ class Goblin {
      */
     void merge(GoblinUltraCircuitBuilder& circuit_builder)
     {
+        BB_OP_COUNT_TIME_NAME("Goblin::merge");
         // Complete the circuit logic by recursively verifying previous merge proof if it exists
         if (merge_proof_exists) {
             RecursiveMergeVerifier merge_verifier{ &circuit_builder };

diff --git a/barretenberg/cpp/src/barretenberg/polynomials/pow.hpp b/barretenberg/cpp/src/barretenberg/polynomials/pow.hpp
@@ -125,7 +125,6 @@ template <typename FF> struct PowPolynomial {
      */
     BB_PROFILE void compute_values()
     {
-        BB_OP_COUNT_TIME();
         size_t pow_size = 1 << betas.size();
         pow_betas = std::vector<FF>(pow_size);
 

diff --git a/...g/cpp/src/barretenberg/proof_system/circuit_builder/goblin_translator_circuit_builder.hpp b/...g/cpp/src/barretenberg/proof_system/circuit_builder/goblin_translator_circuit_builder.hpp
@@ -364,6 +364,7 @@ class GoblinTranslatorCircuitBuilder : public CircuitBuilderBase<bb::fr> {
                                    std::shared_ptr<ECCOpQueue> op_queue)
         : GoblinTranslatorCircuitBuilder(batching_challenge_v_, evaluation_input_x_)
     {
+        BB_OP_COUNT_TIME_NAME("GoblinTranslatorCircuitBuilder::constructor");
         feed_ecc_op_queue_into_circuit(op_queue);
     }
 

diff --git a/barretenberg/cpp/src/barretenberg/protogalaxy/decider_prover.cpp b/barretenberg/cpp/src/barretenberg/protogalaxy/decider_prover.cpp
@@ -1,4 +1,5 @@
 #include "decider_prover.hpp"
+#include "barretenberg/common/op_count.hpp"
 #include "barretenberg/sumcheck/sumcheck.hpp"
 
 namespace bb {
@@ -102,6 +103,8 @@ template <IsUltraFlavor Flavor> HonkProof& DeciderProver_<Flavor>::export_proof(
 
 template <IsUltraFlavor Flavor> HonkProof& DeciderProver_<Flavor>::construct_proof()
 {
+    BB_OP_COUNT_TIME_NAME("Decider::construct_proof");
+
     // Add ϕ, \vec{β*}, e* to transcript
     execute_preamble_round();
 

diff --git a/barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy_prover.cpp b/barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy_prover.cpp
@@ -346,6 +346,7 @@ template <class ProverInstances> void ProtoGalaxyProver_<ProverInstances>::accum
 template <class ProverInstances>
 FoldingResult<typename ProverInstances::Flavor> ProtoGalaxyProver_<ProverInstances>::fold_instances()
 {
+    BB_OP_COUNT_TIME_NAME("ProtogalaxyProver::fold_instances");
     preparation_round();
     perturbator_round();
     combiner_quotient_round();

diff --git a/barretenberg/cpp/src/barretenberg/translator_vm/goblin_translator_composer.cpp b/barretenberg/cpp/src/barretenberg/translator_vm/goblin_translator_composer.cpp
@@ -190,6 +190,7 @@ void GoblinTranslatorComposer::compute_witness(CircuitBuilder& circuit_builder)
 GoblinTranslatorProver GoblinTranslatorComposer::create_prover(CircuitBuilder& circuit_builder,
                                                                const std::shared_ptr<Transcript>& transcript)
 {
+    BB_OP_COUNT_TIME_NAME("GoblinTranslatorComposer::create_prover");
 
     // Compute total number of gates, dyadic circuit size, etc.
     compute_circuit_size_parameters(circuit_builder);

diff --git a/barretenberg/cpp/src/barretenberg/translator_vm/goblin_translator_prover.cpp b/barretenberg/cpp/src/barretenberg/translator_vm/goblin_translator_prover.cpp
@@ -176,6 +176,8 @@ HonkProof& GoblinTranslatorProver::export_proof()
 
 HonkProof& GoblinTranslatorProver::construct_proof()
 {
+    BB_OP_COUNT_TIME_NAME("GoblinTranslatorProver::construct_proof");
+
     // Add circuit size public input size and public inputs to transcript.
     execute_preamble_round();
 

diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/ultra_composer.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/ultra_composer.cpp
@@ -67,6 +67,8 @@ void UltraComposer_<Flavor>::compute_verification_key(const std::shared_ptr<Prov
 template <IsUltraFlavor Flavor>
 std::shared_ptr<ProverInstance_<Flavor>> UltraComposer_<Flavor>::create_instance(CircuitBuilder& circuit)
 {
+    BB_OP_COUNT_TIME_NAME("UltraComposer::create_prover_instance");
+
     circuit.add_gates_to_ensure_all_polys_are_non_zero();
     circuit.finalize_circuit();
     auto instance = std::make_shared<Instance>(circuit);