feat: folding GoblinUltra instances in ProtoGalaxy (AztecProtocol#4340

)

Adds the missing functionality to be able to fold `GoblinUltra`
instances to PG prover and verifier (both native and recursive).

On one hand, this includes committing to the additional witness
polynomials ( ECC op wires and data bus related polynomials) as well as
computing and committing to the log derivativee inverses similar to the
pre-sumcheck rounds in the UltraProver

We also need to add extra functionality to be able to fold linearly
dependent relations. Such relations (in our codebase, just one
subrelation part of the data bus relations) operate on the entire
execution trace rather than a single row. They don't have to hold at
each row in part which implies that we won't multiply them by the pow
polynomial.

This requires us to make modifications just to the perturbator (F
polynomial). The combiner (`G` polynomial) makes use of the `accumulate`
function, which is implemented for every subrelation we have which, in
the case of linearly dependent relations, does not use the
`scaling_factor` provided as argument.

In the perturbator, we accumulate the evaluation of the full honk
relation at each row and return a vector of `FF`. We modify it to add to
the value at index 0 (representing row 0) also the
`linear_dependent_contribution` (the value of the linear dependent
subrelation over the entire execution trace) _batched_ by its
coresponding batching challenge `alpha` (recall we have a batching
challenge for each subrelation). Otherwise than that the protocol
remains unchanged.

Closes AztecProtocol/barretenberg#753.

barretenberg/cpp/src/barretenberg/flavor/goblin_ultra.hpp

@@ -380,36 +380,35 @@ class GoblinUltraFlavor {
             calldata_read_counts = "CALLDATA_READ_COUNTS";
             lookup_inverses = "LOOKUP_INVERSES";
 
-            // The ones beginning with "__" are only used for debugging
-            q_c = "__Q_C";
-            q_l = "__Q_L";
-            q_r = "__Q_R";
-            q_o = "__Q_O";
-            q_4 = "__Q_4";
-            q_m = "__Q_M";
-            q_arith = "__Q_ARITH";
-            q_sort = "__Q_SORT";
-            q_elliptic = "__Q_ELLIPTIC";
-            q_aux = "__Q_AUX";
-            q_lookup = "__Q_LOOKUP";
-            q_busread = "__Q_BUSREAD";
-            q_poseidon2_external = "__Q_POSEIDON2_EXTERNAL";
-            q_poseidon2_internal = "__Q_POSEIDON2_INTERNAL";
-            sigma_1 = "__SIGMA_1";
-            sigma_2 = "__SIGMA_2";
-            sigma_3 = "__SIGMA_3";
-            sigma_4 = "__SIGMA_4";
-            id_1 = "__ID_1";
-            id_2 = "__ID_2";
-            id_3 = "__ID_3";
-            id_4 = "__ID_4";
-            table_1 = "__TABLE_1";
-            table_2 = "__TABLE_2";
-            table_3 = "__TABLE_3";
-            table_4 = "__TABLE_4";
-            lagrange_first = "__LAGRANGE_FIRST";
-            lagrange_last = "__LAGRANGE_LAST";
-            lagrange_ecc_op = "__Q_ECC_OP_QUEUE";
+            q_c = "Q_C";
+            q_l = "Q_L";
+            q_r = "Q_R";
+            q_o = "Q_O";
+            q_4 = "Q_4";
+            q_m = "Q_M";
+            q_arith = "Q_ARITH";
+            q_sort = "Q_SORT";
+            q_elliptic = "Q_ELLIPTIC";
+            q_aux = "Q_AUX";
+            q_lookup = "Q_LOOKUP";
+            q_busread = "Q_BUSREAD";
+            q_poseidon2_external = "Q_POSEIDON2_EXTERNAL";
+            q_poseidon2_internal = "Q_POSEIDON2_INTERNAL";
+            sigma_1 = "SIGMA_1";
+            sigma_2 = "SIGMA_2";
+            sigma_3 = "SIGMA_3";
+            sigma_4 = "SIGMA_4";
+            id_1 = "ID_1";
+            id_2 = "ID_2";
+            id_3 = "ID_3";
+            id_4 = "ID_4";
+            table_1 = "TABLE_1";
+            table_2 = "TABLE_2";
+            table_3 = "TABLE_3";
+            table_4 = "TABLE_4";
+            lagrange_first = "LAGRANGE_FIRST";
+            lagrange_last = "LAGRANGE_LAST";
+            lagrange_ecc_op = "Q_ECC_OP_QUEUE";
         };
     };
 
@@ -458,15 +457,16 @@ class GoblinUltraFlavor {
                 this->w_l = commitments.w_l;
                 this->w_r = commitments.w_r;
                 this->w_o = commitments.w_o;
-                this->sorted_accum = commitments.sorted_accum;
                 this->w_4 = commitments.w_4;
+                this->sorted_accum = commitments.sorted_accum;
                 this->z_perm = commitments.z_perm;
                 this->z_lookup = commitments.z_lookup;
                 this->ecc_op_wire_1 = commitments.ecc_op_wire_1;
                 this->ecc_op_wire_2 = commitments.ecc_op_wire_2;
                 this->ecc_op_wire_3 = commitments.ecc_op_wire_3;
+                this->ecc_op_wire_4 = commitments.ecc_op_wire_4;
                 this->calldata = commitments.calldata;
-                this->calldata = commitments.calldata_read_counts;
+                this->calldata_read_counts = commitments.calldata_read_counts;
                 this->lookup_inverses = commitments.lookup_inverses;
             }
         }

barretenberg/cpp/src/barretenberg/flavor/goblin_ultra_recursive.hpp

@@ -46,7 +46,8 @@ template <typename BuilderType> class GoblinUltraRecursiveFlavor_ {
     using FF = typename Curve::ScalarField;
     using Commitment = typename Curve::Element;
     using CommitmentHandle = typename Curve::Element;
-    using NativeVerificationKey = GoblinUltraFlavor::VerificationKey;
+    using NativeFlavor = GoblinUltraFlavor;
+    using NativeVerificationKey = NativeFlavor::VerificationKey;
 
     // Note(luke): Eventually this may not be needed at all
     using VerifierCommitmentKey = bb::VerifierCommitmentKey<Curve>;

barretenberg/cpp/src/barretenberg/flavor/ultra_recursive.hpp

@@ -54,7 +54,8 @@ template <typename BuilderType> class UltraRecursiveFlavor_ {
     using Commitment = typename Curve::Element;
     using CommitmentHandle = typename Curve::Element;
     using FF = typename Curve::ScalarField;
-    using NativeVerificationKey = UltraFlavor::VerificationKey;
+    using NativeFlavor = UltraFlavor;
+    using NativeVerificationKey = NativeFlavor::VerificationKey;
 
     // Note(luke): Eventually this may not be needed at all
     using VerifierCommitmentKey = bb::VerifierCommitmentKey<Curve>;

barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy_prover.cpp

@@ -34,6 +34,27 @@ void ProtoGalaxyProver_<ProverInstances>::finalise_and_send_instance(std::shared
         transcript->send_to_verifier(domain_separator + "_" + wire_labels[idx], wire_comms[idx]);
     }
 
+    if constexpr (IsGoblinFlavor<Flavor>) {
+        // Commit to Goblin ECC op wires
+        witness_commitments.ecc_op_wire_1 = commitment_key->commit(instance->proving_key->ecc_op_wire_1);
+        witness_commitments.ecc_op_wire_2 = commitment_key->commit(instance->proving_key->ecc_op_wire_2);
+        witness_commitments.ecc_op_wire_3 = commitment_key->commit(instance->proving_key->ecc_op_wire_3);
+        witness_commitments.ecc_op_wire_4 = commitment_key->commit(instance->proving_key->ecc_op_wire_4);
+
+        auto op_wire_comms = instance->witness_commitments.get_ecc_op_wires();
+        auto labels = commitment_labels.get_ecc_op_wires();
+        for (size_t idx = 0; idx < Flavor::NUM_WIRES; ++idx) {
+            transcript->send_to_verifier(domain_separator + "_" + labels[idx], op_wire_comms[idx]);
+        }
+        // Commit to DataBus columns
+        witness_commitments.calldata = commitment_key->commit(instance->proving_key->calldata);
+        witness_commitments.calldata_read_counts = commitment_key->commit(instance->proving_key->calldata_read_counts);
+        transcript->send_to_verifier(domain_separator + "_" + commitment_labels.calldata,
+                                     instance->witness_commitments.calldata);
+        transcript->send_to_verifier(domain_separator + "_" + commitment_labels.calldata_read_counts,
+                                     instance->witness_commitments.calldata_read_counts);
+    }
+
     auto eta = transcript->get_challenge(domain_separator + "_eta");
     instance->compute_sorted_accumulator_polynomials(eta);
 
@@ -47,6 +68,16 @@ void ProtoGalaxyProver_<ProverInstances>::finalise_and_send_instance(std::shared
     transcript->send_to_verifier(domain_separator + "_" + commitment_labels.w_4, witness_commitments.w_4);
 
     auto [beta, gamma] = transcript->get_challenges(domain_separator + "_beta", domain_separator + "_gamma");
+
+    if constexpr (IsGoblinFlavor<Flavor>) {
+        // Compute and commit to the logderivative inverse used in DataBus
+        instance->compute_logderivative_inverse(beta, gamma);
+        instance->witness_commitments.lookup_inverses =
+            commitment_key->commit(instance->prover_polynomials.lookup_inverses);
+        transcript->send_to_verifier(domain_separator + "_" + commitment_labels.lookup_inverses,
+                                     instance->witness_commitments.lookup_inverses);
+    }
+
     instance->compute_grand_product_polynomials(beta, gamma);
 
     witness_commitments.z_perm = commitment_key->commit(instance->prover_polynomials.z_perm);
@@ -303,7 +334,6 @@ FoldingResult<typename ProverInstances::Flavor> ProtoGalaxyProver_<ProverInstanc
         compute_next_accumulator(instances, combiner_quotient, combiner_challenge, compressed_perturbator);
     res.folding_data = transcript->proof_data;
     res.accumulator = next_accumulator;
-
     return res;
 }
 

barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy_prover.hpp

@@ -127,16 +127,23 @@ template <class ProverInstances_> class ProtoGalaxyProver_ {
     std::shared_ptr<Instance> get_accumulator() { return instances[0]; }
 
     /**
-     * @brief Compute the values of the full Honk relation at each row in the execution trace, f_i(ω) in the
-     * ProtoGalaxy paper, given the evaluations of all the prover polynomials and α (the parameter that helps establish
-     * each subrelation is independently valid in Honk - from the Plonk paper, DO NOT confuse with α in ProtoGalaxy),
+     * @brief Compute the values of the full Honk relation at each row in the execution trace, representing f_i(ω) in
+     * the ProtoGalaxy paper, given the evaluations of all the prover polynomials and \vec{α} (the batching challenges
+     * that help establishing each subrelation is independently valid in Honk - from the Plonk paper, DO NOT confuse
+     * with α in ProtoGalaxy).
+     *
+     * @details When folding GoblinUltra instances, one of the relations is linearly dependent. We define such relations
+     * as acting on the entire execution trace and hence requiring to be accumulated separately as we iterate over each
+     * row. At the end of the function, the linearly dependent contribution is accumulated at index 0 representing the
+     * sum f_0(ω) + α_j*g(ω) where f_0 represents the full honk evaluation at row 0, g(ω) is the linearly dependent
+     * subrelation and α_j is its corresponding batching challenge.
      */
     static std::vector<FF> compute_full_honk_evaluations(const ProverPolynomials& instance_polynomials,
                                                          const RelationSeparator& alpha,
                                                          const RelationParameters<FF>& relation_parameters)
     {
         auto instance_size = instance_polynomials.get_polynomial_size();
-
+        FF linearly_dependent_contribution = FF(0);
         std::vector<FF> full_honk_evaluations(instance_size);
         for (size_t row = 0; row < instance_size; row++) {
             auto row_evaluations = instance_polynomials.get_row(row);
@@ -150,17 +157,22 @@ template <class ProverInstances_> class ProtoGalaxyProver_ {
 
             auto output = FF(0);
             auto running_challenge = FF(1);
-            Utils::scale_and_batch_elements(relation_evaluations, alpha, running_challenge, output);
+
+            // Sum relation evaluations, batched by their corresponding relation separator challenge, to get the value
+            // of the full honk relation at a specific row
+            Utils::scale_and_batch_elements(
+                relation_evaluations, alpha, running_challenge, output, linearly_dependent_contribution);
 
             full_honk_evaluations[row] = output;
         }
+        full_honk_evaluations[0] += linearly_dependent_contribution;
         return full_honk_evaluations;
     }
 
     /**
-     * @brief  Recursively compute the parent nodes of each level in there, starting from the leaves. Note that at each
-     * level, the resulting parent nodes will be polynomials of degree (level + 1) because we multiply by an additional
-     * factor of X.
+     * @brief  Recursively compute the parent nodes of each level in the tree, starting from the leaves. Note that at
+     * each level, the resulting parent nodes will be polynomials of degree (level+1) because we multiply by an
+     * additional factor of X.
      */
     static std::vector<FF> construct_coefficients_tree(const std::vector<FF>& betas,
                                                        const std::vector<FF>& deltas,
@@ -307,7 +319,8 @@ template <class ProverInstances_> class ProtoGalaxyProver_ {
                 FF pow_challenge = pow_betas[idx];
 
                 // Accumulate the i-th row's univariate contribution. Note that the relation parameters passed to this
-                // function have already been folded
+                // function have already been folded. Moreover, linear-dependent relations that act over the entire
+                // execution trace rather than on rows, will not be multiplied by the pow challenge.
                 accumulate_relation_univariates(
                     thread_univariate_accumulators[thread_idx],
                     extended_univariates[thread_idx],
@@ -323,6 +336,7 @@ template <class ProverInstances_> class ProtoGalaxyProver_ {
         // Batch the univariate contributions from each sub-relation to obtain the round univariate
         return batch_over_relations(univariate_accumulators, instances.alphas);
     }
+
     static ExtendedUnivariateWithRandomization batch_over_relations(TupleOfTuplesOfUnivariates& univariate_accumulators,
                                                                     const CombinedRelationSeparator& alpha)
     {
@@ -331,7 +345,7 @@ template <class ProverInstances_> class ProtoGalaxyProver_ {
         auto result = std::get<0>(std::get<0>(univariate_accumulators))
                           .template extend_to<ProverInstances::BATCHED_EXTENDED_LENGTH>();
         size_t idx = 0;
-        auto scale_and_sum = [&]<size_t outer_idx, size_t>(auto& element) {
+        auto scale_and_sum = [&]<size_t outer_idx, size_t inner_idx>(auto& element) {
             auto extended = element.template extend_to<ProverInstances::BATCHED_EXTENDED_LENGTH>();
             extended *= alpha[idx];
             result += extended;
@@ -416,7 +430,8 @@ template <class ProverInstances_> class ProtoGalaxyProver_ {
     }
 
     /**
-     * @brief Compute the next accumulator (ϕ*, ω*\vec{\beta*}, e*), send the public data ϕ*  and the folding parameters
+     * @brief Compute the next accumulator (ϕ*, ω*, \vec{\beta*}, e*), send the public data ϕ*  and the folding
+     * parameters
      * (\vec{\beta*}, e*) to the verifier and return the complete accumulator
      *
      * @details At this stage, we assume that the instances have the same size and the same number of public parameter.s

barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy_verifier.cpp

@@ -87,6 +87,22 @@ void ProtoGalaxyVerifier_<VerifierInstances>::receive_and_finalise_instance(cons
     witness_commitments.w_r = transcript->template receive_from_prover<Commitment>(domain_separator + "_" + labels.w_r);
     witness_commitments.w_o = transcript->template receive_from_prover<Commitment>(domain_separator + "_" + labels.w_o);
 
+    if constexpr (IsGoblinFlavor<Flavor>) {
+        // Get  commitments to the ECC wire polynomials and databus polynomials
+        witness_commitments.ecc_op_wire_1 =
+            transcript->template receive_from_prover<Commitment>(domain_separator + "_" + labels.ecc_op_wire_1);
+        witness_commitments.ecc_op_wire_2 =
+            transcript->template receive_from_prover<Commitment>(domain_separator + "_" + labels.ecc_op_wire_2);
+        witness_commitments.ecc_op_wire_3 =
+            transcript->template receive_from_prover<Commitment>(domain_separator + "_" + labels.ecc_op_wire_3);
+        witness_commitments.ecc_op_wire_4 =
+            transcript->template receive_from_prover<Commitment>(domain_separator + "_" + labels.ecc_op_wire_4);
+        witness_commitments.calldata =
+            transcript->template receive_from_prover<Commitment>(domain_separator + "_" + labels.calldata);
+        witness_commitments.calldata_read_counts =
+            transcript->template receive_from_prover<Commitment>(domain_separator + "_" + labels.calldata_read_counts);
+    }
+
     // Get challenge for sorted list batching and wire four memory records commitment
     auto eta = transcript->get_challenge(domain_separator + "_eta");
     witness_commitments.sorted_accum =
@@ -95,6 +111,13 @@ void ProtoGalaxyVerifier_<VerifierInstances>::receive_and_finalise_instance(cons
 
     // Get permutation challenges and commitment to permutation and lookup grand products
     auto [beta, gamma] = transcript->get_challenges(domain_separator + "_beta", domain_separator + "_gamma");
+
+    if constexpr (IsGoblinFlavor<Flavor>) {
+        // If Goblin (i.e. using DataBus) receive commitments to log-deriv inverses polynomial
+        witness_commitments.lookup_inverses = transcript->template receive_from_prover<Commitment>(
+            domain_separator + "_" + commitment_labels.lookup_inverses);
+    }
+
     witness_commitments.z_perm =
         transcript->template receive_from_prover<Commitment>(domain_separator + "_" + labels.z_perm);
     witness_commitments.z_lookup =