feat: Prototype native merkle trees (#4457)

This PR contains work around understanding the performance of our hash functions and merkle trees. 1. Parameterise our existing MerkleTree and NulliferTree to accept any compatible hash function 2. Creates a new native AppendOnlyTree only allowing addition of leaves at the end 3. Creates a new multi-threaded native IndexedMerkleTree. 4. Add tests to validate new trees against previous ones 5. Adds benchmarks for hash functions and new trees. 6. Exposes poseidon hash functions over the WASM and benchmarks it. ### Benchmarks **Hash Functions** **Pedersen - Native** ``` ------------------------------------------------------------------------------------------ Benchmark Time CPU Iterations ------------------------------------------------------------------------------------------ native_pedersen_hash_pair_bench/min_time:3.000 0.178 ms 0.178 ms 23450 ``` **Pedersen - Single hash over WASM 1000 times** Executed 1000 hashes at an average 593.7273874282837us / hash **Pedersen - 1024 hashes over WASM 10 times** Executed 10240 hashes at an average 581.7138370126486us / hash **Poseidon - Native** ``` ---------------------------------------------------------------------------------- Benchmark Time CPU Iterations ---------------------------------------------------------------------------------- poseiden_hash_bench 0.012 ms 0.012 ms 57022 ``` **Poseidon - Single hash over WASM 1000 times** Executed 1000 hashes at an average 48.4412202835083us / hash **Poseidon - 1024 hashes over WASM 10 times** Executed 10240 hashes at an average 37.44762847200036us / hash ### Merkle Trees The following benchmarks follow the format of {benchmark-type}/{batch-insertion-size}/{iteration-count}. **Append Only Tree** ``` ------------------------------------------------------------------------------------------------ Benchmark Time CPU Iterations ------------------------------------------------------------------------------------------------ append_only_tree_bench<Pedersen>/2/iterations:100 5.75 ms 5.75 ms 100 append_only_tree_bench<Pedersen>/4/iterations:100 6.02 ms 6.02 ms 100 append_only_tree_bench<Pedersen>/8/iterations:100 6.65 ms 6.65 ms 100 append_only_tree_bench<Pedersen>/16/iterations:100 7.89 ms 7.89 ms 100 append_only_tree_bench<Pedersen>/32/iterations:100 10.7 ms 10.7 ms 100 append_only_tree_bench<Pedersen>/64/iterations:100 16.3 ms 16.3 ms 100 append_only_tree_bench<Pedersen>/128/iterations:100 27.8 ms 27.8 ms 100 append_only_tree_bench<Poseidon2>/2/iterations:1000 0.395 ms 0.395 ms 1000 append_only_tree_bench<Poseidon2>/4/iterations:1000 0.407 ms 0.408 ms 1000 append_only_tree_bench<Poseidon2>/8/iterations:1000 0.454 ms 0.454 ms 1000 append_only_tree_bench<Poseidon2>/16/iterations:1000 0.531 ms 0.531 ms 1000 append_only_tree_bench<Poseidon2>/32/iterations:1000 0.726 ms 0.726 ms 1000 append_only_tree_bench<Poseidon2>/64/iterations:1000 1.11 ms 1.11 ms 1000 append_only_tree_bench<Poseidon2>/128/iterations:1000 1.89 ms 1.89 ms 1000 ``` **Indexed Tree** ``` ---------------------------------------------------------------------------------------------------------- Benchmark Time CPU Iterations ---------------------------------------------------------------------------------------------------------- single_thread_indexed_tree_bench<Pedersen>/2/iterations:100 18.2 ms 18.2 ms 100 single_thread_indexed_tree_bench<Pedersen>/4/iterations:100 31.3 ms 31.3 ms 100 single_thread_indexed_tree_bench<Pedersen>/8/iterations:100 57.2 ms 57.2 ms 100 single_thread_indexed_tree_bench<Pedersen>/16/iterations:100 108 ms 108 ms 100 single_thread_indexed_tree_bench<Pedersen>/32/iterations:100 210 ms 210 ms 100 single_thread_indexed_tree_bench<Pedersen>/64/iterations:100 415 ms 415 ms 100 single_thread_indexed_tree_bench<Pedersen>/128/iterations:100 824 ms 824 ms 100 multi_thread_indexed_tree_bench<Pedersen>/2/iterations:100 12.8 ms 6.65 ms 100 multi_thread_indexed_tree_bench<Pedersen>/4/iterations:100 14.8 ms 7.31 ms 100 multi_thread_indexed_tree_bench<Pedersen>/8/iterations:100 18.5 ms 8.80 ms 100 multi_thread_indexed_tree_bench<Pedersen>/16/iterations:100 23.9 ms 12.0 ms 100 multi_thread_indexed_tree_bench<Pedersen>/32/iterations:100 35.1 ms 18.5 ms 100 multi_thread_indexed_tree_bench<Pedersen>/64/iterations:100 59.3 ms 31.6 ms 100 multi_thread_indexed_tree_bench<Pedersen>/128/iterations:100 109 ms 66.5 ms 100 single_thread_indexed_tree_bench<Poseidon2>/2/iterations:1000 1.26 ms 1.26 ms 1000 single_thread_indexed_tree_bench<Poseidon2>/4/iterations:1000 2.15 ms 2.15 ms 1000 single_thread_indexed_tree_bench<Poseidon2>/8/iterations:1000 3.88 ms 3.88 ms 1000 single_thread_indexed_tree_bench<Poseidon2>/16/iterations:1000 7.41 ms 7.41 ms 1000 single_thread_indexed_tree_bench<Poseidon2>/32/iterations:1000 14.6 ms 14.6 ms 1000 single_thread_indexed_tree_bench<Poseidon2>/64/iterations:1000 28.7 ms 28.7 ms 1000 single_thread_indexed_tree_bench<Poseidon2>/128/iterations:1000 56.8 ms 56.8 ms 1000 multi_thread_indexed_tree_bench<Poseidon2>/2/iterations:1000 1.01 ms 0.812 ms 1000 multi_thread_indexed_tree_bench<Poseidon2>/4/iterations:1000 1.17 ms 0.865 ms 1000 multi_thread_indexed_tree_bench<Poseidon2>/8/iterations:1000 1.46 ms 0.979 ms 1000 multi_thread_indexed_tree_bench<Poseidon2>/16/iterations:1000 2.05 ms 1.23 ms 1000 multi_thread_indexed_tree_bench<Poseidon2>/32/iterations:1000 3.24 ms 1.70 ms 1000 multi_thread_indexed_tree_bench<Poseidon2>/64/iterations:1000 5.84 ms 2.54 ms 1000 multi_thread_indexed_tree_bench<Poseidon2>/128/iterations:1000 13.3 ms 6.36 ms 1000 ``` --------- Co-authored-by: ludamad <adam@aztecprotocol.com>
AztecProtocol · Feb 9, 2024 · 7d5e056 · 7d5e056
1 parent 7b519a4
commit 7d5e056
Show file tree

Hide file tree

Showing 77 changed files with 2,743 additions and 735 deletions.
diff --git a/barretenberg/cpp/scripts/stdlib-tests b/barretenberg/cpp/scripts/stdlib-tests
@@ -3,7 +3,6 @@ stdlib_aes128_tests
 stdlib_blake2s_tests
 stdlib_blake3s_tests
 stdlib_ecdsa_tests
-stdlib_merkle_tree_tests
 stdlib_pedersen_commitment_tests
 stdlib_pedersen_hash_tests
 stdlib_poseidon2_tests

diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt
@@ -127,7 +127,7 @@ set(BARRETENBERG_TARGET_OBJECTS
     $<TARGET_OBJECTS:stdlib_blake2s_objects>
     $<TARGET_OBJECTS:stdlib_blake3s_objects>
     $<TARGET_OBJECTS:stdlib_keccak_objects>
-    $<TARGET_OBJECTS:stdlib_merkle_tree_objects>
+    $<TARGET_OBJECTS:crypto_merkle_tree_objects>
     $<TARGET_OBJECTS:stdlib_pedersen_commitment_objects>
     $<TARGET_OBJECTS:stdlib_pedersen_hash_objects>
     $<TARGET_OBJECTS:stdlib_poseidon2_objects>

diff --git a/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt
@@ -7,5 +7,9 @@ add_subdirectory(pippenger_bench)
 add_subdirectory(plonk_bench)
 add_subdirectory(protogalaxy_bench)
 add_subdirectory(relations_bench)
+add_subdirectory(widgets_bench)
+add_subdirectory(poseidon2_bench)
+add_subdirectory(merkle_tree_bench)
+add_subdirectory(indexed_tree_bench)
+add_subdirectory(append_only_tree_bench)
 add_subdirectory(ultra_bench)
-add_subdirectory(widgets_bench)
diff --git a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt
@@ -0,0 +1 @@
+barretenberg_module(append_only_tree_bench crypto_poseidon2 crypto_merkle_tree)
diff --git a/...etenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp b/...etenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp
@@ -0,0 +1,54 @@
+#include "barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp"
+#include "barretenberg/crypto/merkle_tree/array_store.hpp"
+#include "barretenberg/crypto/merkle_tree/hash.hpp"
+#include "barretenberg/numeric/random/engine.hpp"
+#include <benchmark/benchmark.h>
+
+using namespace benchmark;
+using namespace bb::crypto::merkle_tree;
+
+using Pedersen = AppendOnlyTree<ArrayStore, PedersenHashPolicy>;
+using Poseidon2 = AppendOnlyTree<ArrayStore, Poseidon2HashPolicy>;
+
+const size_t TREE_DEPTH = 32;
+const size_t MAX_BATCH_SIZE = 128;
+
+namespace {
+auto& random_engine = bb::numeric::get_randomness();
+} // namespace
+
+template <typename TreeType> void perform_batch_insert(TreeType& tree, const std::vector<fr>& values)
+{
+    tree.add_values(values);
+}
+
+template <typename TreeType> void append_only_tree_bench(State& state) noexcept
+{
+    const size_t batch_size = size_t(state.range(0));
+    const size_t depth = TREE_DEPTH;
+
+    ArrayStore store(depth, 1024 * 1024);
+    TreeType tree = TreeType(store, depth);
+
+    for (auto _ : state) {
+        state.PauseTiming();
+        std::vector<fr> values(batch_size);
+        for (size_t i = 0; i < batch_size; ++i) {
+            values[i] = fr(random_engine.get_random_uint256());
+        }
+        state.ResumeTiming();
+        perform_batch_insert(tree, values);
+    }
+}
+BENCHMARK(append_only_tree_bench<Pedersen>)
+    ->Unit(benchmark::kMillisecond)
+    ->RangeMultiplier(2)
+    ->Range(2, MAX_BATCH_SIZE)
+    ->Iterations(100);
+BENCHMARK(append_only_tree_bench<Poseidon2>)
+    ->Unit(benchmark::kMillisecond)
+    ->RangeMultiplier(2)
+    ->Range(2, MAX_BATCH_SIZE)
+    ->Iterations(1000);
+
+BENCHMARK_MAIN();
diff --git a/barretenberg/cpp/src/barretenberg/benchmark/goblin_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/goblin_bench/CMakeLists.txt
@@ -1 +1 @@
-barretenberg_module(goblin_bench ultra_honk eccvm stdlib_recursion stdlib_sha256 stdlib_merkle_tree stdlib_primitives)
+barretenberg_module(goblin_bench ultra_honk eccvm stdlib_recursion stdlib_sha256 crypto_merkle_tree stdlib_primitives)
diff --git a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt
@@ -0,0 +1 @@
+barretenberg_module(indexed_tree_bench crypto_poseidon2 crypto_merkle_tree)
diff --git a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp
@@ -0,0 +1,87 @@
+#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp"
+#include "barretenberg/crypto/merkle_tree/array_store.hpp"
+#include "barretenberg/crypto/merkle_tree/hash.hpp"
+#include "barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp"
+#include "barretenberg/numeric/random/engine.hpp"
+#include <benchmark/benchmark.h>
+
+using namespace benchmark;
+using namespace bb::crypto::merkle_tree;
+
+using Poseidon2 = IndexedTree<ArrayStore, LeavesCache, Poseidon2HashPolicy>;
+using Pedersen = IndexedTree<ArrayStore, LeavesCache, PedersenHashPolicy>;
+
+const size_t TREE_DEPTH = 32;
+const size_t MAX_BATCH_SIZE = 128;
+
+namespace {
+auto& random_engine = bb::numeric::get_randomness();
+} // namespace
+
+template <typename TreeType>
+void perform_batch_insert(TreeType& tree, const std::vector<fr>& values, bool single_threaded)
+{
+    tree.add_or_update_values(values, single_threaded);
+}
+
+template <typename TreeType> void multi_thread_indexed_tree_bench(State& state) noexcept
+{
+    const size_t batch_size = size_t(state.range(0));
+    const size_t depth = TREE_DEPTH;
+
+    ArrayStore store(depth, 1024 * 1024);
+    TreeType tree = TreeType(store, depth, batch_size);
+
+    for (auto _ : state) {
+        state.PauseTiming();
+        std::vector<fr> values(batch_size);
+        for (size_t i = 0; i < batch_size; ++i) {
+            values[i] = fr(random_engine.get_random_uint256());
+        }
+        state.ResumeTiming();
+        perform_batch_insert(tree, values, false);
+    }
+}
+
+template <typename TreeType> void single_thread_indexed_tree_bench(State& state) noexcept
+{
+    const size_t batch_size = size_t(state.range(0));
+    const size_t depth = TREE_DEPTH;
+
+    ArrayStore store(depth, 1024 * 1024);
+    TreeType tree = TreeType(store, depth, batch_size);
+
+    for (auto _ : state) {
+        state.PauseTiming();
+        std::vector<fr> values(batch_size);
+        for (size_t i = 0; i < batch_size; ++i) {
+            values[i] = fr(random_engine.get_random_uint256());
+        }
+        state.ResumeTiming();
+        perform_batch_insert(tree, values, true);
+    }
+}
+BENCHMARK(single_thread_indexed_tree_bench<Pedersen>)
+    ->Unit(benchmark::kMillisecond)
+    ->RangeMultiplier(2)
+    ->Range(2, MAX_BATCH_SIZE)
+    ->Iterations(100);
+
+BENCHMARK(multi_thread_indexed_tree_bench<Pedersen>)
+    ->Unit(benchmark::kMillisecond)
+    ->RangeMultiplier(2)
+    ->Range(2, MAX_BATCH_SIZE)
+    ->Iterations(100);
+
+BENCHMARK(single_thread_indexed_tree_bench<Poseidon2>)
+    ->Unit(benchmark::kMillisecond)
+    ->RangeMultiplier(2)
+    ->Range(2, MAX_BATCH_SIZE)
+    ->Iterations(1000);
+BENCHMARK(multi_thread_indexed_tree_bench<Poseidon2>)
+    ->Unit(benchmark::kMillisecond)
+    ->RangeMultiplier(2)
+    ->Range(2, MAX_BATCH_SIZE)
+    ->Iterations(1000);
+
+BENCHMARK_MAIN();
diff --git a/barretenberg/cpp/src/barretenberg/benchmark/ivc_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/ivc_bench/CMakeLists.txt
@@ -1 +1 @@
-barretenberg_module(ivc_bench client_ivc stdlib_recursion stdlib_sha256 stdlib_merkle_tree stdlib_primitives)
+barretenberg_module(ivc_bench client_ivc stdlib_recursion stdlib_sha256 crypto_merkle_tree stdlib_primitives)
diff --git a/barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/CMakeLists.txt
@@ -0,0 +1 @@
+barretenberg_module(merkle_tree_bench crypto_poseidon2 crypto_merkle_tree)
diff --git a/.../stdlib/merkle_tree/merkle_tree.bench.cpp → ...k/merkle_tree_bench/merkle_tree.bench.cpp b/.../stdlib/merkle_tree/merkle_tree.bench.cpp → ...k/merkle_tree_bench/merkle_tree.bench.cpp
@@ -1,15 +1,17 @@
-#include "merkle_tree.hpp"
+#include "barretenberg/crypto/merkle_tree/merkle_tree.hpp"
+#include "barretenberg/crypto/merkle_tree/hash.hpp"
+#include "barretenberg/crypto/merkle_tree/memory_store.hpp"
 #include "barretenberg/numeric/random/engine.hpp"
-#include "hash.hpp"
-#include "memory_store.hpp"
 #include <benchmark/benchmark.h>
 
 using namespace benchmark;
-using namespace bb::stdlib::merkle_tree;
+using namespace bb::crypto::merkle_tree;
+
+using TreeType = MerkleTree<MemoryStore, PedersenHashPolicy>;
 
 namespace {
 auto& engine = bb::numeric::get_debug_randomness();
-}
+} // namespace
 
 constexpr size_t DEPTH = 256;
 constexpr size_t MAX = 4096;
@@ -33,7 +35,7 @@ BENCHMARK(hash)->MinTime(5);
 void update_first_element(State& state) noexcept
 {
     MemoryStore store;
-    MerkleTree<MemoryStore> db(store, DEPTH);
+    TreeType db(store, DEPTH);
 
     for (auto _ : state) {
         db.update_element(0, VALUES[1]);
@@ -46,7 +48,7 @@ void update_elements(State& state) noexcept
     for (auto _ : state) {
         state.PauseTiming();
         MemoryStore store;
-        MerkleTree<MemoryStore> db(store, DEPTH);
+        TreeType db(store, DEPTH);
         state.ResumeTiming();
         for (size_t i = 0; i < (size_t)state.range(0); ++i) {
             db.update_element(i, VALUES[i]);
@@ -60,10 +62,10 @@ void update_random_elements(State& state) noexcept
     for (auto _ : state) {
         state.PauseTiming();
         MemoryStore store;
-        MerkleTree db(store, DEPTH);
+        TreeType db(store, DEPTH);
         for (size_t i = 0; i < (size_t)state.range(0); i++) {
             state.PauseTiming();
-            auto index = MerkleTree<MemoryStore>::index_t(engine.get_random_uint256());
+            auto index = TreeType::index_t(engine.get_random_uint256());
             state.ResumeTiming();
             db.update_element(index, VALUES[i]);
         }

diff --git a/barretenberg/cpp/src/barretenberg/benchmark/poseidon2_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/poseidon2_bench/CMakeLists.txt
@@ -0,0 +1 @@
+barretenberg_module(poseidon2_bench crypto_poseidon2)
diff --git a/...berg/crypto/poseidon2/poseidon2.bench.cpp → ...hmark/poseidon2_bench/poseidon2.bench.cpp b/...berg/crypto/poseidon2/poseidon2.bench.cpp → ...hmark/poseidon2_bench/poseidon2.bench.cpp
@@ -1,4 +1,4 @@
-#include "./poseidon2.hpp"
+#include "barretenberg/crypto/poseidon2/poseidon2.hpp"
 #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp"
 #include <benchmark/benchmark.h>
 
@@ -24,4 +24,20 @@ void native_poseidon2_commitment_bench(State& state) noexcept
 }
 BENCHMARK(native_poseidon2_commitment_bench)->Arg(10)->Arg(1000)->Arg(10000);
 
+grumpkin::fq poseiden_hash_impl(const grumpkin::fq& x, const grumpkin::fq& y)
+{
+    std::vector<grumpkin::fq> to_hash{ x, y };
+    return bb::crypto::Poseidon2<bb::crypto::Poseidon2Bn254ScalarFieldParams>::hash(to_hash);
+}
+
+void poseiden_hash_bench(State& state) noexcept
+{
+    grumpkin::fq x = grumpkin::fq::random_element();
+    grumpkin::fq y = grumpkin::fq::random_element();
+    for (auto _ : state) {
+        DoNotOptimize(poseiden_hash_impl(x, y));
+    }
+}
+BENCHMARK(poseiden_hash_bench)->Unit(benchmark::kMillisecond);
+
 BENCHMARK_MAIN();
diff --git a/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt
@@ -2,5 +2,5 @@ barretenberg_module(ultra_bench
   ultra_honk
   stdlib_sha256
   stdlib_keccak
-  stdlib_merkle_tree
+  crypto_merkle_tree
   stdlib_recursion)
diff --git a/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/mock_proofs.hpp b/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/mock_proofs.hpp
@@ -2,17 +2,17 @@
 #include <benchmark/benchmark.h>
 #include <cstddef>
 
+#include "barretenberg/crypto/merkle_tree/membership.hpp"
+#include "barretenberg/crypto/merkle_tree/memory_store.hpp"
+#include "barretenberg/crypto/merkle_tree/memory_tree.hpp"
+#include "barretenberg/crypto/merkle_tree/merkle_tree.hpp"
 #include "barretenberg/goblin/mock_circuits.hpp"
 #include "barretenberg/plonk/composer/standard_composer.hpp"
 #include "barretenberg/plonk/composer/ultra_composer.hpp"
 #include "barretenberg/proof_system/types/circuit_type.hpp"
 #include "barretenberg/stdlib/encryption/ecdsa/ecdsa.hpp"
 #include "barretenberg/stdlib/hash/keccak/keccak.hpp"
 #include "barretenberg/stdlib/hash/sha256/sha256.hpp"
-#include "barretenberg/stdlib/merkle_tree/membership.hpp"
-#include "barretenberg/stdlib/merkle_tree/memory_store.hpp"
-#include "barretenberg/stdlib/merkle_tree/memory_tree.hpp"
-#include "barretenberg/stdlib/merkle_tree/merkle_tree.hpp"
 #include "barretenberg/stdlib/primitives/bool/bool.hpp"
 #include "barretenberg/stdlib/primitives/curves/secp256k1.hpp"
 #include "barretenberg/stdlib/primitives/field/field.hpp"

diff --git a/barretenberg/cpp/src/barretenberg/crypto/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/crypto/CMakeLists.txt
@@ -9,4 +9,5 @@ add_subdirectory(schnorr)
 add_subdirectory(sha256)
 add_subdirectory(ecdsa)
 add_subdirectory(aes128)
-add_subdirectory(poseidon2)
+add_subdirectory(poseidon2)
+add_subdirectory(merkle_tree)
diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt
@@ -0,0 +1 @@
+barretenberg_module(crypto_merkle_tree stdlib_primitives stdlib_blake3s stdlib_pedersen_hash)