refactor(public/avm): from hints to the end of the world (#13459)

I had a few design goals with this PR
1. Move the merkle hinting layer to the merkle db directly (from the
PublicContractsDB)
1. Make hinting a TxSimulator concept
1. Divide the tree accesses into low level and high level (like we do in
C++).
1. Tighten our exports
1. Some cleanups

I'm satisfied with the result, but there are some things worth noting
* Now the TxSimulator wraps the merkle db into a hinting merkle db. This
means that the hints will _always_ be generated and the caller has no
control. However, reexecution and the TXE could benefit from a model
where no hinting (or tracing) is done. I had a version of this PR where
this was done via an extra PublicTreesDBFactory that was passed to the
TxSimulator, but I found it too much for the moment. I think at some
point we can consider having a more flexible factory type for the
PublicProcessor and the TxSimulator where we can specify, e.g., merkle
ops, tracing, hinting, etc and have a "Proving" factory and a
"Simulation-only" factory.

## AI generated description

This PR refactors the public processor and simulator components to use
`MerkleTreeWriteOperations` directly instead of going through the
`PublicTreesDB` abstraction. Key changes include:

- Changed `PublicTxSimulator` to accept a `MerkleTreeWriteOperations`
instead of `PublicTreesDB`
- Moved `PublicTreesDB` to be an internal implementation detail rather
than a public API
- Refactored `HintingPublicTreesDB` to `HintingMerkleWriteOperations` to
work directly with the merkle tree interface
- Moved `SimpleContractDataSource` from `avm/fixtures` to
`public/fixtures` for better organization
- Added high-level methods to `PublicTreesDB` for writing note hashes
and nullifiers
- Simplified tree operations in `PublicPersistableStateManager` by using
the new high-level methods
- Updated `checkL1ToL2MessageExists` to no longer require a contract
address parameter

This change simplifies the codebase by reducing unnecessary abstractions
and making the component dependencies more explicit.

Author: fcarreiro

yarn-project/end-to-end/src/e2e_block_building.test.ts

@@ -26,14 +26,9 @@ import { TestContract } from '@aztec/noir-contracts.js/Test';
 import { TokenContract } from '@aztec/noir-contracts.js/Token';
 import type { SequencerClient } from '@aztec/sequencer-client';
 import type { TestSequencerClient } from '@aztec/sequencer-client/test';
-import {
-  PublicContractsDB,
-  PublicProcessorFactory,
-  type PublicTreesDB,
-  type PublicTxResult,
-  TelemetryPublicTxSimulator,
-} from '@aztec/simulator/server';
+import { type PublicContractsDB, PublicProcessorFactory, TelemetryPublicTxSimulator } from '@aztec/simulator/server';
 import type { AztecNodeAdmin } from '@aztec/stdlib/interfaces/client';
+import type { MerkleTreeWriteOperations } from '@aztec/stdlib/trees';
 import { TX_ERROR_EXISTING_NULLIFIER, type Tx } from '@aztec/stdlib/tx';
 
 import { jest } from '@jest/globals';
@@ -631,19 +626,19 @@ async function sendAndWait(calls: ContractFunctionInteraction[]) {
 const TEST_PUBLIC_TX_SIMULATION_DELAY_MS = 300;
 
 class TestPublicTxSimulator extends TelemetryPublicTxSimulator {
-  public override async simulate(tx: Tx): Promise<PublicTxResult> {
+  public override async simulate(tx: Tx) {
     await sleep(TEST_PUBLIC_TX_SIMULATION_DELAY_MS);
     return super.simulate(tx);
   }
 }
 class TestPublicProcessorFactory extends PublicProcessorFactory {
   protected override createPublicTxSimulator(
-    treesDB: PublicTreesDB,
+    merkleTree: MerkleTreeWriteOperations,
     contractsDB: PublicContractsDB,
     globalVariables: GlobalVariables,
     doMerkleOperations: boolean,
     skipFeeEnforcement: boolean,
   ) {
-    return new TestPublicTxSimulator(treesDB, contractsDB, globalVariables, doMerkleOperations, skipFeeEnforcement);
+    return new TestPublicTxSimulator(merkleTree, contractsDB, globalVariables, doMerkleOperations, skipFeeEnforcement);
   }
 }

yarn-project/prover-client/src/mocks/test_context.ts

@@ -7,11 +7,9 @@ import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree';
 import { protocolContractTreeRoot } from '@aztec/protocol-contracts';
 import { computeFeePayerBalanceLeafSlot } from '@aztec/protocol-contracts/fee-juice';
 import {
-  PublicContractsDB,
   PublicProcessor,
-  PublicTreesDB,
+  PublicProcessorFactory,
   PublicTxSimulationTester,
-  PublicTxSimulator,
   SimpleContractDataSource,
 } from '@aztec/simulator/server';
 import { PublicDataWrite } from '@aztec/stdlib/avm';
@@ -40,7 +38,6 @@ export class TestContext {
   private feePayerBalance: Fr;
 
   constructor(
-    public publicTxSimulator: PublicTxSimulator,
     public worldState: MerkleTreeAdminDatabase,
     public publicProcessor: PublicProcessor,
     public globalVariables: GlobalVariables,
@@ -79,26 +76,17 @@ export class TestContext {
 
     // Separated dbs for public processor and prover - see public_processor for context
     const ws = await NativeWorldStateService.tmp(
-      undefined /* rollupAddress */,
-      true /* cleanupTmpDir */,
+      /*rollupAddress=*/ undefined,
+      /*cleanupTmpDir=*/ true,
       prefilledPublicData,
     );
     const merkleTrees = await ws.fork();
 
     const contractDataSource = new SimpleContractDataSource();
-    const treesDB = new PublicTreesDB(merkleTrees);
-    const contractsDB = new PublicContractsDB(contractDataSource);
-
     const tester = new PublicTxSimulationTester(merkleTrees, contractDataSource);
 
-    const publicTxSimulator = new PublicTxSimulator(treesDB, contractsDB, globalVariables, true);
-    const processor = new PublicProcessor(
-      globalVariables,
-      treesDB,
-      contractsDB,
-      publicTxSimulator,
-      new TestDateProvider(),
-    );
+    const processorFactory = new PublicProcessorFactory(contractDataSource, new TestDateProvider());
+    const processor = processorFactory.create(merkleTrees, globalVariables, /*skipFeeEnforcement=*/ false);
 
     let localProver: ServerCircuitProver;
     const config = await getEnvironmentConfig(logger);
@@ -127,7 +115,6 @@ export class TestContext {
     facade.start();
 
     return new this(
-      publicTxSimulator,
       ws,
       processor,
       globalVariables,

yarn-project/simulator/src/public/avm/avm_simulator.test.ts

@@ -12,21 +12,18 @@ import { Fq, Fr, Point } from '@aztec/foundation/fields';
 import type { Fieldable } from '@aztec/foundation/serialize';
 import { CounterContract } from '@aztec/noir-contracts.js/Counter';
 import { type FunctionArtifact, FunctionSelector } from '@aztec/stdlib/abi';
-import { PublicDataWrite } from '@aztec/stdlib/avm';
 import { AztecAddress } from '@aztec/stdlib/aztec-address';
 import { SerializableContractInstance, computePublicBytecodeCommitment } from '@aztec/stdlib/contract';
 import { GasFees } from '@aztec/stdlib/gas';
 import {
   computeNoteHashNonce,
-  computePublicDataTreeLeafSlot,
   computeUniqueNoteHash,
   computeVarArgsHash,
   siloNoteHash,
   siloNullifier,
 } from '@aztec/stdlib/hash';
 import { PublicKeys } from '@aztec/stdlib/keys';
 import { makeContractClassPublic, makeContractInstanceFromClassId } from '@aztec/stdlib/testing';
-import { MerkleTreeId } from '@aztec/stdlib/trees';
 import { NativeWorldStateService } from '@aztec/world-state';
 
 import { strict as assert } from 'assert';
@@ -35,6 +32,7 @@ import { mock } from 'jest-mock-extended';
 
 import { SideEffectTrace } from '../../public/side_effect_trace.js';
 import type { PublicSideEffectTraceInterface } from '../../public/side_effect_trace_interface.js';
+import { SimpleContractDataSource } from '../fixtures/simple_contract_data_source.js';
 import { PublicContractsDB, PublicTreesDB } from '../public_db_sources.js';
 import type { PublicPersistableStateManager } from '../state_manager/state_manager.js';
 import type { AvmContext } from './avm_context.js';
@@ -57,7 +55,6 @@ import {
   resolveAvmTestContractAssertionMessage,
   resolveContractAssertionMessage,
 } from './fixtures/index.js';
-import { SimpleContractDataSource } from './fixtures/simple_contract_data_source.js';
 import {
   Add,
   CalldataCopy,
@@ -1112,10 +1109,7 @@ describe('AVM simulator: transpiled Noir contracts', () => {
     let trace: PublicSideEffectTraceInterface;
     let persistableState: PublicPersistableStateManager;
 
-    let leafSlot0: Fr;
-
     beforeAll(async () => {
-      leafSlot0 = await computePublicDataTreeLeafSlot(address, slot0);
       siloedNullifier0 = await siloNullifier(address, value0);
       const siloedNoteHash0 = await siloNoteHash(address, value0);
       const nonce = await computeNoteHashNonce(firstNullifier, noteHashIndexInTx);
@@ -1161,8 +1155,7 @@ describe('AVM simulator: transpiled Noir contracts', () => {
         expect(trace.traceNewNoteHash).toHaveBeenCalledWith(uniqueNoteHash0);
       });
       it('Note hash check properly returns exists=false', async () => {
-        const treeInfo = await treesDB.getTreeInfo(MerkleTreeId.NOTE_HASH_TREE);
-        const leafIndex = treeInfo.size;
+        const leafIndex = (await treesDB.getTreeSnapshots()).noteHashTree.nextAvailableLeafIndex;
 
         const calldata = [uniqueNoteHash0, new Fr(leafIndex)];
         const context = createContext(calldata);
@@ -1173,9 +1166,8 @@ describe('AVM simulator: transpiled Noir contracts', () => {
         expect(results.output).toEqual([/*exists=*/ Fr.ZERO]);
       });
       it('Note hash check properly returns exists=true', async () => {
-        const treeInfo = await treesDB.getTreeInfo(MerkleTreeId.NOTE_HASH_TREE);
-        const leafIndex = treeInfo.size;
-        await treesDB.appendLeaves(MerkleTreeId.NOTE_HASH_TREE, [uniqueNoteHash0]);
+        const leafIndex = (await treesDB.getTreeSnapshots()).noteHashTree.nextAvailableLeafIndex;
+        await treesDB.writeNoteHash(uniqueNoteHash0);
 
         const calldata = [uniqueNoteHash0, new Fr(leafIndex)];
         const context = createContext(calldata);
@@ -1210,7 +1202,7 @@ describe('AVM simulator: transpiled Noir contracts', () => {
       });
       it('Nullifier check properly returns exists=true', async () => {
         const calldata = [value0];
-        await treesDB.sequentialInsert(MerkleTreeId.NULLIFIER_TREE, [siloedNullifier0.toBuffer()]);
+        await treesDB.writeNullifier(siloedNullifier0);
         const context = createContext(calldata);
         const bytecode = getAvmTestContractBytecode('nullifier_exists');
 
@@ -1245,8 +1237,7 @@ describe('AVM simulator: transpiled Noir contracts', () => {
 
       it('Should read value in storage (single) - written before, leaf exists', async () => {
         const context = createContext();
-        const publicDataWrite = new PublicDataWrite(leafSlot0, value0);
-        await treesDB.sequentialInsert(MerkleTreeId.PUBLIC_DATA_TREE, [publicDataWrite.toBuffer()]);
+        await treesDB.storageWrite(context.environment.address, slot0, value0);
 
         const bytecode = getAvmTestContractBytecode('read_storage_single');
 

yarn-project/simulator/src/public/avm/fixtures/avm_simulation_tester.ts

@@ -16,11 +16,11 @@ import {
   initExecutionEnvironment,
   resolveContractAssertionMessage,
 } from '../../avm/fixtures/index.js';
+import { SimpleContractDataSource } from '../../fixtures/simple_contract_data_source.js';
 import { PublicContractsDB, PublicTreesDB } from '../../public_db_sources.js';
 import { PublicPersistableStateManager } from '../../state_manager/state_manager.js';
 import { AvmSimulator } from '../avm_simulator.js';
 import { BaseAvmSimulationTester } from './base_avm_simulation_tester.js';
-import { SimpleContractDataSource } from './simple_contract_data_source.js';
 
 const TIMESTAMP = new Fr(99833);
 const DEFAULT_GAS_FEES = new GasFees(2, 3);

yarn-project/simulator/src/public/avm/fixtures/base_avm_simulation_tester.ts

@@ -11,8 +11,8 @@ import { computePublicDataTreeLeafSlot, siloNullifier } from '@aztec/stdlib/hash
 import type { MerkleTreeWriteOperations } from '@aztec/stdlib/interfaces/server';
 import { MerkleTreeId } from '@aztec/stdlib/trees';
 
+import type { SimpleContractDataSource } from '../../fixtures/simple_contract_data_source.js';
 import { createContractClassAndInstance } from './index.js';
-import type { SimpleContractDataSource } from './simple_contract_data_source.js';
 
 /**
  * An abstract test class that enables tests of real apps in the AVM without requiring e2e tests.

yarn-project/simulator/src/public/avm/index.ts

@@ -1,2 +1,2 @@
 export * from './avm_simulator.js';
-export * from './fixtures/simple_contract_data_source.js';
+export * from '../fixtures/simple_contract_data_source.js';

yarn-project/simulator/src/public/avm/opcodes/accrued_substate.ts

@@ -181,11 +181,7 @@ export class L1ToL2MessageExists extends Instruction {
 
     const msgHash = memory.get(msgHashOffset).toFr();
     const msgLeafIndex = memory.get(msgLeafIndexOffset).toFr();
-    const exists = await context.persistableState.checkL1ToL2MessageExists(
-      context.environment.address,
-      msgHash,
-      msgLeafIndex,
-    );
+    const exists = await context.persistableState.checkL1ToL2MessageExists(msgHash, msgLeafIndex);
     memory.set(existsOffset, exists ? new Uint1(1) : new Uint1(0));
   }
 }

yarn-project/simulator/src/common/debug_fn_name.ts renamed to yarn-project/simulator/src/public/debug_fn_name.ts

@@ -2,7 +2,7 @@ import type { Fr } from '@aztec/foundation/fields';
 import { FunctionSelector } from '@aztec/stdlib/abi';
 import type { AztecAddress } from '@aztec/stdlib/aztec-address';
 
-import type { PublicContractsDBInterface } from '../public/db_interfaces.js';
+import type { PublicContractsDBInterface } from './db_interfaces.js';
 
 export async function getPublicFunctionDebugName(
   db: PublicContractsDBInterface,

yarn-project/simulator/src/public/fixtures/index.ts

@@ -1,2 +1,3 @@
 export * from './public_tx_simulation_tester.js';
 export * from './utils.js';
+export * from './simple_contract_data_source.js';

yarn-project/simulator/src/public/fixtures/public_tx_simulation_tester.ts

@@ -10,11 +10,11 @@ import { NativeWorldStateService } from '@aztec/world-state';
 
 import { BaseAvmSimulationTester } from '../avm/fixtures/base_avm_simulation_tester.js';
 import { DEFAULT_BLOCK_NUMBER, getContractFunctionAbi, getFunctionSelector } from '../avm/fixtures/index.js';
-import { SimpleContractDataSource } from '../avm/fixtures/simple_contract_data_source.js';
-import { PublicContractsDB, PublicTreesDB } from '../public_db_sources.js';
+import { PublicContractsDB } from '../public_db_sources.js';
 import { MeasuredPublicTxSimulator } from '../public_tx_simulator/measured_public_tx_simulator.js';
 import type { PublicTxResult } from '../public_tx_simulator/public_tx_simulator.js';
 import { TestExecutorMetrics } from '../test_executor_metrics.js';
+import { SimpleContractDataSource } from './simple_contract_data_source.js';
 import { createTxForPublicCalls } from './utils.js';
 
 const TIMESTAMP = new Fr(99833);
@@ -47,11 +47,9 @@ export class PublicTxSimulationTester extends BaseAvmSimulationTester {
   ) {
     super(contractDataSource, merkleTree);
 
-    const treesDB = new PublicTreesDB(merkleTree);
     const contractsDB = new PublicContractsDB(contractDataSource);
-
     this.simulator = new MeasuredPublicTxSimulator(
-      treesDB,
+      merkleTree,
       contractsDB,
       globals,
       /*doMerkleOperations=*/ true,

yarn-project/simulator/src/public/avm/fixtures/simple_contract_data_source.ts renamed to yarn-project/simulator/src/public/fixtures/simple_contract_data_source.ts

@@ -4,7 +4,7 @@ import type { ContractArtifact, FunctionSelector } from '@aztec/stdlib/abi';
 import type { AztecAddress } from '@aztec/stdlib/aztec-address';
 import type { ContractClassPublic, ContractDataSource, ContractInstanceWithAddress } from '@aztec/stdlib/contract';
 
-import { getFunctionSelector } from './index.js';
+import { getFunctionSelector } from '../avm/fixtures/index.js';
 
 /**
  * This class is used during public/avm testing to function as a database of