feat: Refactor IPA claim handling in acir format to support them for AVM (#13547)

Refactor where we handle IPA claim handling to be at the acir_format
level instead in within `process_honk_recursion_constraints`. This is
needed to handle circuits like the public base which will have AVM
recursion constraints and RollupHonk recursion constraints, which will
now BOTH produce IPA claims/proofs.

Fix Goblin AVM recursive verifier test and clean some small type-related
things.

Author: lucasxia01

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph_description_ultra_recursive_verifier.test.cpp

@@ -46,7 +46,7 @@ template <typename RecursiveFlavor> class BoomerangRecursiveVerifierTest : publi
     using VerificationKey = typename RecursiveVerifier::VerificationKey;
 
     using AggState = aggregation_state<OuterBuilder>;
-    using VerifierOutput = bb::stdlib::recursion::honk::UltraRecursiveVerifierOutput<RecursiveFlavor>;
+    using VerifierOutput = bb::stdlib::recursion::honk::UltraRecursiveVerifierOutput<OuterBuilder>;
 
     /**
      * @brief Create a non-trivial arbitrary inner circuit, the proof of which will be recursively verified
@@ -125,7 +125,7 @@ template <typename RecursiveFlavor> class BoomerangRecursiveVerifierTest : publi
         pairing_points.P1.x.fix_witness();
         pairing_points.P1.y.fix_witness();
         if constexpr (HasIPAAccumulator<OuterFlavor>) {
-            output.ipa_opening_claim.set_public();
+            output.ipa_claim.set_public();
             outer_circuit.ipa_proof = convert_stdlib_proof_to_native(output.ipa_proof);
         }
         info("Recursive Verifier: num gates = ", outer_circuit.get_estimated_num_finalized_gates());

barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_format.cpp

@@ -270,29 +270,4 @@ template <typename Builder> class GateCounter {
     size_t prev_gate_count{};
 };
 
-void process_plonk_recursion_constraints(Builder& builder,
-                                         AcirFormat& constraint_system,
-                                         bool has_valid_witness_assignments,
-                                         GateCounter<Builder>& gate_counter);
-void process_honk_recursion_constraints(Builder& builder,
-                                        AcirFormat& constraint_system,
-                                        bool has_valid_witness_assignments,
-                                        GateCounter<Builder>& gate_counter,
-                                        uint32_t honk_recursion);
-
-void process_ivc_recursion_constraints(MegaCircuitBuilder& builder,
-                                       AcirFormat& constraints,
-                                       ClientIVC* ivc,
-                                       bool has_valid_witness_assignments,
-                                       GateCounter<MegaCircuitBuilder>& gate_counter);
-
-#ifndef DISABLE_AZTEC_VM
-stdlib::recursion::aggregation_state<Builder> process_avm_recursion_constraints(
-    Builder& builder,
-    AcirFormat& constraint_system,
-    bool has_valid_witness_assignments,
-    GateCounter<Builder>& gate_counter,
-    stdlib::recursion::aggregation_state<Builder> current_aggregation_object);
-#endif
-
 } // namespace acir_format

barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_format.hpp

@@ -224,12 +224,10 @@ HonkRecursionConstraintOutput<Builder> create_avm2_recursion_constraints_goblin(
     // Execute the Goblin AVM2 recursive verifier
     RecursiveVerifier verifier(builder, key_fields);
 
-    auto output_agg_object = verifier.verify_proof(
+    bb::avm2::AvmGoblinRecursiveVerifier::RecursiveAvmGoblinOutput output = verifier.verify_proof(
         proof_fields, bb::avm2::PublicInputs::flat_to_columns(public_inputs_flattened), input_aggregation_object);
 
-    return { .agg_obj = output_agg_object.aggregation_object,
-             .ipa_claim = output_agg_object.ipa_claim,
-             .ipa_proof = output_agg_object.ipa_proof };
+    return output;
 }
 
 } // namespace acir_format

barretenberg/cpp/src/barretenberg/dsl/acir_format/avm2_recursion_constraint.cpp

@@ -159,7 +159,7 @@ TEST_F(AcirAvm2RecursionConstraint, TestGenerateVKFromConstraintsWithoutWitness)
     std::shared_ptr<OuterVerificationKey> expected_vk;
     {
         AcirProgram avm_verifier_program = construct_avm_verifier_program({ avm_prover_output });
-        const ProgramMetadata metadata{ .honk_recursion = 1 };
+        const ProgramMetadata metadata{ .honk_recursion = 2 };
         auto layer_2_circuit = create_circuit(avm_verifier_program, metadata);
 
         info("circuit gates = ", layer_2_circuit.get_estimated_num_finalized_gates());
@@ -182,7 +182,7 @@ TEST_F(AcirAvm2RecursionConstraint, TestGenerateVKFromConstraintsWithoutWitness)
 
         // Clear the program witness then construct the bberg circuit as normal
         avm_verifier_program.witness.clear();
-        const ProgramMetadata metadata{ .honk_recursion = 1 };
+        const ProgramMetadata metadata{ .honk_recursion = 2 };
         auto layer_2_circuit = create_circuit(avm_verifier_program, metadata);
 
         info("circuit gates = ", layer_2_circuit.get_estimated_num_finalized_gates());

barretenberg/cpp/src/barretenberg/dsl/acir_format/avm2_recursion_constraint.test.cpp

@@ -264,14 +264,14 @@ HonkRecursionConstraintOutput<typename Flavor::CircuitBuilder> create_honk_recur
     auto vkey = std::make_shared<RecursiveVerificationKey>(builder, key_fields);
     RecursiveVerifier verifier(&builder, vkey);
     HonkRecursionConstraintOutput<Builder> output;
-    UltraRecursiveVerifierOutput<Flavor> verifier_output = verifier.verify_proof(proof_fields, input_agg_obj);
+    UltraRecursiveVerifierOutput<Builder> verifier_output = verifier.verify_proof(proof_fields, input_agg_obj);
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/996): investigate whether assert_equal on public inputs
     // is important, like what the plonk recursion constraint does.
 
     output.agg_obj = verifier_output.agg_obj;
     if constexpr (HasIPAAccumulator<Flavor>) {
         ASSERT(HasIPAAccumulator<Flavor>);
-        output.ipa_claim = verifier_output.ipa_opening_claim;
+        output.ipa_claim = verifier_output.ipa_claim;
         output.ipa_proof = verifier_output.ipa_proof;
     }
     return output;

barretenberg/cpp/src/barretenberg/dsl/acir_format/honk_recursion_constraint.cpp

@@ -2,6 +2,7 @@
 #include "barretenberg/commitment_schemes/claim.hpp"
 #include "barretenberg/dsl/acir_format/recursion_constraint.hpp"
 #include "barretenberg/honk/proof_system/types/proof.hpp"
+#include "barretenberg/stdlib/honk_verifier/ultra_recursive_verifier.hpp"
 #include "barretenberg/stdlib/plonk_recursion/aggregation_state/aggregation_state.hpp"
 #include "barretenberg/stdlib/primitives/bigfield/bigfield.hpp"
 #include "barretenberg/stdlib/primitives/curves/grumpkin.hpp"
@@ -11,11 +12,8 @@ namespace acir_format {
 
 using namespace bb;
 
-template <typename Builder> struct HonkRecursionConstraintOutput {
-    stdlib::recursion::aggregation_state<Builder> agg_obj;
-    OpeningClaim<stdlib::grumpkin<Builder>> ipa_claim;
-    StdlibProof<Builder> ipa_proof;
-};
+template <typename Builder>
+using HonkRecursionConstraintOutput = bb::stdlib::recursion::honk::UltraRecursiveVerifierOutput<Builder>;
 
 template <typename Flavor>
 HonkRecursionConstraintOutput<typename Flavor::CircuitBuilder> create_honk_recursion_constraints(

barretenberg/cpp/src/barretenberg/dsl/acir_format/honk_recursion_constraint.hpp

@@ -81,7 +81,7 @@ class IvcRecursionConstraintTest : public ::testing::Test {
 
         {
             using RecursiveFlavor = UltraRecursiveFlavor_<Builder>;
-            using VerifierOutput = bb::stdlib::recursion::honk::UltraRecursiveVerifierOutput<RecursiveFlavor>;
+            using VerifierOutput = bb::stdlib::recursion::honk::UltraRecursiveVerifierOutput<Builder>;
             using OuterAggState = bb::stdlib::recursion::aggregation_state<Builder>;
 
             // Create an arbitrary inner circuit

barretenberg/cpp/src/barretenberg/dsl/acir_format/ivc_recursion_constraint.test.cpp

@@ -136,11 +136,11 @@ TEST_F(ClientIVCRecursionTests, ClientTubeBase)
     auto base_vk = std::make_shared<RollupFlavor::VerificationKey>(&base_builder, tube_vk);
     auto base_tube_proof = bb::convert_native_proof_to_stdlib(&base_builder, native_tube_proof);
     UltraRecursiveVerifier base_verifier{ &base_builder, base_vk };
-    UltraRecursiveVerifierOutput<RollupFlavor> output =
+    UltraRecursiveVerifierOutput<Builder> output =
         base_verifier.verify_proof(base_tube_proof, AggregationObject::construct_default(base_builder));
     info("Tube UH Recursive Verifier: num prefinalized gates = ", base_builder.num_gates);
     output.agg_obj.set_public();
-    output.ipa_opening_claim.set_public();
+    output.ipa_claim.set_public();
     base_builder.ipa_proof = tube_prover.proving_key->proving_key.ipa_proof;
     EXPECT_EQ(base_builder.failed(), false) << base_builder.err();
     EXPECT_TRUE(CircuitChecker::check(base_builder));
@@ -149,7 +149,7 @@ TEST_F(ClientIVCRecursionTests, ClientTubeBase)
     auto base_proving_key = std::make_shared<DeciderProvingKey_<NativeFlavor>>(base_builder);
     auto ipa_transcript = std::make_shared<NativeTranscript>(base_proving_key->proving_key.ipa_proof);
     IPA<curve::Grumpkin>::reduce_verify(
-        ipa_verification_key, output.ipa_opening_claim.get_native_opening_claim(), ipa_transcript);
+        ipa_verification_key, output.ipa_claim.get_native_opening_claim(), ipa_transcript);
 }
 
 // Ensure that the Client IVC Recursive Verifier Circuit does not depend on the Client IVC input

barretenberg/cpp/src/barretenberg/stdlib/client_ivc_verifier/client_ivc_recursive_verifier.test.cpp

@@ -142,7 +142,7 @@ UltraRecursiveVerifier_<Flavor>::Output UltraRecursiveVerifier_<Flavor>::verify_
         if (verification_key->verification_key->contains_ipa_claim) {
             PublicComponentKey ipa_claim_key{ verification_key->verification_key->ipa_claim_public_input_indices[0],
                                               true };
-            output.ipa_opening_claim = PublicIpaClaim::reconstruct(verification_key->public_inputs, ipa_claim_key);
+            output.ipa_claim = PublicIpaClaim::reconstruct(verification_key->public_inputs, ipa_claim_key);
         }
     }
 

barretenberg/cpp/src/barretenberg/stdlib/honk_verifier/ultra_recursive_verifier.cpp

@@ -11,10 +11,9 @@
 
 namespace bb::stdlib::recursion::honk {
 
-template <typename Flavor> struct UltraRecursiveVerifierOutput {
-    using Builder = typename Flavor::CircuitBuilder;
+template <typename Builder> struct UltraRecursiveVerifierOutput {
     aggregation_state<Builder> agg_obj;
-    OpeningClaim<grumpkin<Builder>> ipa_opening_claim;
+    OpeningClaim<grumpkin<Builder>> ipa_claim;
     StdlibProof<Builder> ipa_proof;
 };
 template <typename Flavor> class UltraRecursiveVerifier_ {
@@ -31,7 +30,7 @@ template <typename Flavor> class UltraRecursiveVerifier_ {
     using AggregationObject = aggregation_state<Builder>;
     using Transcript = bb::BaseTranscript<bb::stdlib::recursion::honk::StdlibTranscriptParams<Builder>>;
     using OinkVerifier = OinkRecursiveVerifier_<Flavor>;
-    using Output = UltraRecursiveVerifierOutput<Flavor>;
+    using Output = UltraRecursiveVerifierOutput<Builder>;
 
     explicit UltraRecursiveVerifier_(Builder* builder,
                                      const std::shared_ptr<NativeVerificationKey>& native_verifier_key);

barretenberg/cpp/src/barretenberg/stdlib/honk_verifier/ultra_recursive_verifier.hpp

@@ -44,7 +44,7 @@ template <typename RecursiveFlavor> class RecursiveVerifierTest : public testing
     using VerificationKey = typename RecursiveVerifier::VerificationKey;
 
     using AggState = aggregation_state<OuterBuilder>;
-    using VerifierOutput = bb::stdlib::recursion::honk::UltraRecursiveVerifierOutput<RecursiveFlavor>;
+    using VerifierOutput = bb::stdlib::recursion::honk::UltraRecursiveVerifierOutput<OuterBuilder>;
     /**
      * @brief Create a non-trivial arbitrary inner circuit, the proof of which will be recursively verified
      *
@@ -163,7 +163,7 @@ template <typename RecursiveFlavor> class RecursiveVerifierTest : public testing
             typename RecursiveVerifier::Output verifier_output =
                 verifier.verify_proof(inner_proof, AggState::construct_default(outer_circuit));
             if constexpr (HasIPAAccumulator<OuterFlavor>) {
-                verifier_output.ipa_opening_claim.set_public();
+                verifier_output.ipa_claim.set_public();
                 outer_circuit.ipa_proof = convert_stdlib_proof_to_native(verifier_output.ipa_proof);
             }
 
@@ -204,7 +204,7 @@ template <typename RecursiveFlavor> class RecursiveVerifierTest : public testing
         VerifierOutput output = verifier.verify_proof(inner_proof, agg_obj);
         AggState pairing_points = output.agg_obj;
         if constexpr (HasIPAAccumulator<OuterFlavor>) {
-            output.ipa_opening_claim.set_public();
+            output.ipa_claim.set_public();
             outer_circuit.ipa_proof = convert_stdlib_proof_to_native(output.ipa_proof);
         }
         info("Recursive Verifier: num gates = ", outer_circuit.get_estimated_num_finalized_gates());

barretenberg/cpp/src/barretenberg/stdlib/honk_verifier/ultra_recursive_verifier.test.cpp

@@ -52,11 +52,7 @@ class AvmGoblinRecursiveVerifier {
 
     // The structure of the final output of the goblinized AVM2 recursive verifier. The IPA data comes from recursive
     // verification of the ECCVM proof as part of Goblin recursive verification.
-    struct RecursiveAvmGoblinOutput {
-        std::vector<UltraFF> ipa_proof;
-        OpeningClaim<stdlib::grumpkin<UltraBuilder>> ipa_claim;
-        stdlib::recursion::aggregation_state<UltraBuilder> aggregation_object;
-    };
+    using RecursiveAvmGoblinOutput = stdlib::recursion::honk::UltraRecursiveVerifierOutput<UltraBuilder>;
 
     // Output of prover for inner Mega-arithmetized AVM recursive verifier circuit; input to the outer verifier
     struct InnerProverOutput {
@@ -145,26 +141,17 @@ class AvmGoblinRecursiveVerifier {
         GoblinRecursiveVerifier goblin_verifier{ &ultra_builder, inner_output.goblin_vk };
         GoblinRecursiveVerifierOutput goblin_verifier_output = goblin_verifier.verify(inner_output.goblin_proof);
 
-        // Propagate the IPA claim via the public inputs of the outer circuit
-        // TODO(https://github.com/AztecProtocol/barretenberg/issues/1306): Determine the right location/entity to
-        // handle this IPA data propagation.
-
-        // TODO(jeamon): Restore ipa_claim and ipa_proof integration in the builder. While running ./bootstrap.sh,
-        // for the mock-protocol-circuits, we hit an assertion at circuit_builder_base_impl.hpp:262 ("added IPA claim
-        // when one already exists")
-        // ultra_builder.add_ipa_claim(goblin_verifier_output.opening_claim.get_witness_indices());
-        // ultra_builder.ipa_proof = convert_stdlib_proof_to_native(goblin_verifier_output.ipa_transcript->proof_data);
-        // ASSERT(ultra_builder.ipa_proof.size() && "IPA proof should not be empty");
-
         // Validate the consistency of the AVM2 verifier inputs {\pi, pub_inputs, VK}_{AVM2} between the inner (Mega)
         // circuit and the outer (Ultra) by asserting equality on the independently computed hashes of this data.
         const FF ultra_hash = stdlib::poseidon2<UltraBuilder>::hash(ultra_builder, hash_buffer);
         mega_proof[inner_output.mega_hash_public_input_index].assert_equal(ultra_hash);
 
         // Return ipa proof, ipa claim and output aggregation object produced from verifying the Mega + Goblin proofs
-        return RecursiveAvmGoblinOutput{ .ipa_proof = goblin_verifier_output.ipa_transcript->proof_data,
-                                         .ipa_claim = goblin_verifier_output.opening_claim,
-                                         .aggregation_object = mega_verifier_output.agg_obj };
+        return RecursiveAvmGoblinOutput{
+            .agg_obj = mega_verifier_output.agg_obj,
+            .ipa_claim = goblin_verifier_output.opening_claim,
+            .ipa_proof = goblin_verifier_output.ipa_transcript->proof_data,
+        };
     }
 
     /**

barretenberg/cpp/src/barretenberg/vm2/constraining/recursion/goblin_avm_recursive_verifier.hpp

@@ -159,9 +159,6 @@ TEST_F(AvmRecursiveTests, StandardRecursion)
  */
 TEST_F(AvmRecursiveTests, GoblinRecursion)
 {
-    GTEST_SKIP(); // TODO: Fix this test. At the moment, we hit an assertion in the UltraProver.
-                  // Assertion `(proving_key->proving_key.ipa_proof.size() == IPA_PROOF_LENGTH)' failed.
-
     // Type aliases specific to GoblinRecursion test
     using AvmRecursiveVerifier = AvmGoblinRecursiveVerifier;
     using UltraRollupRecursiveFlavor = UltraRollupRecursiveFlavor_<UltraRollupFlavor::CircuitBuilder>;
@@ -204,9 +201,12 @@ TEST_F(AvmRecursiveTests, GoblinRecursion)
         return avm_rec_verifier.verify_proof(stdlib_proof, public_inputs_ct, agg_object);
     }();
 
+    verifier_output.ipa_claim.set_public();
+    outer_circuit.ipa_proof = convert_stdlib_proof_to_native(verifier_output.ipa_proof);
+
     // Ensure that the pairing check is satisfied on the outputs of the recursive verifier
     bool agg_output_valid = verification_key->pcs_verification_key->pairing_check(
-        verifier_output.aggregation_object.P0.get_value(), verifier_output.aggregation_object.P1.get_value());
+        verifier_output.agg_obj.P0.get_value(), verifier_output.agg_obj.P1.get_value());
     ASSERT_TRUE(agg_output_valid) << "Pairing points (aggregation state) are not valid.";
     ASSERT_FALSE(outer_circuit.failed()) << "Outer circuit has failed.";