feat: track rewards and slots (#13546)

This PR adds metrics to track rewards for sequencers/provers and slots
for sequencers.

Author: alexghr

yarn-project/aztec/src/cli/cmds/start_prover_node.ts

@@ -119,6 +119,6 @@ export async function startProverNode(
 
   signalHandlers.push(proverNode.stop.bind(proverNode));
 
-  proverNode.start();
+  await proverNode.start();
   return { config: proverConfig };
 }

yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts

@@ -309,7 +309,7 @@ export class FullProverTest {
       },
       { prefilledPublicData },
     );
-    this.proverNode.start();
+    await this.proverNode.start();
 
     this.logger.warn(`Proofs are now enabled`);
     return this;

yarn-project/end-to-end/src/fixtures/utils.ts

@@ -822,7 +822,7 @@ export async function createAndSyncProverNode(
     { prefilledPublicData },
   );
   getLogger().info(`Created and synced prover node`, { publisherAddress: l1TxUtils.walletClient.account.address });
-  proverNode.start();
+  await proverNode.start();
   return proverNode;
 }
 

yarn-project/prover-node/src/job/epoch-proving-job.test.ts

@@ -13,7 +13,7 @@ import { getTelemetryClient } from '@aztec/telemetry-client';
 
 import { type MockProxy, mock } from 'jest-mock-extended';
 
-import { ProverNodeMetrics } from '../metrics.js';
+import { ProverNodeJobMetrics } from '../metrics.js';
 import type { ProverNodePublisher } from '../prover-node-publisher.js';
 import type { EpochProvingJobData } from './epoch-proving-job-data.js';
 import { EpochProvingJob } from './epoch-proving-job.js';
@@ -25,7 +25,7 @@ describe('epoch-proving-job', () => {
   let l2BlockSource: MockProxy<L2BlockSource>;
   let worldState: MockProxy<WorldStateSynchronizer>;
   let publicProcessorFactory: MockProxy<PublicProcessorFactory>;
-  let metrics: ProverNodeMetrics;
+  let metrics: ProverNodeJobMetrics;
 
   // Created by a dependency
   let db: MockProxy<MerkleTreeWriteOperations>;
@@ -74,7 +74,10 @@ describe('epoch-proving-job', () => {
     publicProcessorFactory = mock<PublicProcessorFactory>();
     db = mock<MerkleTreeWriteOperations>();
     publicProcessor = mock<PublicProcessor>();
-    metrics = new ProverNodeMetrics(getTelemetryClient());
+    metrics = new ProverNodeJobMetrics(
+      getTelemetryClient().getMeter('EpochProvingJob'),
+      getTelemetryClient().getTracer('EpochProvingJob'),
+    );
 
     publicInputs = RootRollupPublicInputs.random();
     proof = Proof.empty();

yarn-project/prover-node/src/job/epoch-proving-job.ts

@@ -15,7 +15,7 @@ import { Attributes, type Traceable, type Tracer, trackSpan } from '@aztec/telem
 
 import * as crypto from 'node:crypto';
 
-import type { ProverNodeMetrics } from '../metrics.js';
+import type { ProverNodeJobMetrics } from '../metrics.js';
 import type { ProverNodePublisher } from '../prover-node-publisher.js';
 import { type EpochProvingJobData, validateEpochProvingJobData } from './epoch-proving-job-data.js';
 
@@ -42,13 +42,13 @@ export class EpochProvingJob implements Traceable {
     private publicProcessorFactory: PublicProcessorFactory,
     private publisher: Pick<ProverNodePublisher, 'submitEpochProof'>,
     private l2BlockSource: L2BlockSource | undefined,
-    private metrics: ProverNodeMetrics,
+    private metrics: ProverNodeJobMetrics,
     private deadline: Date | undefined,
     private config: { parallelBlockLimit: number } = { parallelBlockLimit: 32 },
   ) {
     validateEpochProvingJobData(data);
     this.uuid = crypto.randomUUID();
-    this.tracer = metrics.client.getTracer('EpochProvingJob');
+    this.tracer = metrics.tracer;
   }
 
   public getId(): string {

yarn-project/prover-node/src/metrics.ts

@@ -1,107 +1,187 @@
+import type { RollupContract } from '@aztec/ethereum';
+import type { EthAddress } from '@aztec/foundation/eth-address';
 import { createLogger } from '@aztec/foundation/log';
 import type { L1PublishProofStats, L1PublishStats } from '@aztec/stdlib/stats';
 import {
   Attributes,
+  type BatchObservableResult,
   type Gauge,
   type Histogram,
+  type Meter,
   Metrics,
+  type ObservableGauge,
   type TelemetryClient,
+  type Tracer,
   type UpDownCounter,
   ValueType,
 } from '@aztec/telemetry-client';
 
-import { formatEther } from 'viem';
+import { formatEther, formatUnits } from 'viem';
 
-export class ProverNodeMetrics {
+export class ProverNodeJobMetrics {
   proverEpochExecutionDuration: Histogram;
   provingJobDuration: Histogram;
   provingJobBlocks: Gauge;
   provingJobTransactions: Gauge;
 
-  gasPrice: Histogram;
-  txCount: UpDownCounter;
-  txDuration: Histogram;
-  txGas: Histogram;
-  txCalldataSize: Histogram;
-  txCalldataGas: Histogram;
-  txBlobDataGasUsed: Histogram;
-  txBlobDataGasCost: Histogram;
-  txTotalFee: Histogram;
-
-  private senderBalance: Gauge;
-
   constructor(
-    public readonly client: TelemetryClient,
-    name = 'ProverNode',
+    private meter: Meter,
+    public readonly tracer: Tracer,
     private logger = createLogger('prover-node:publisher:metrics'),
   ) {
-    const meter = client.getMeter(name);
-    this.proverEpochExecutionDuration = meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, {
+    this.proverEpochExecutionDuration = this.meter.createHistogram(Metrics.PROVER_NODE_EXECUTION_DURATION, {
       description: 'Duration of execution of an epoch by the prover',
       unit: 'ms',
       valueType: ValueType.INT,
     });
-    this.provingJobDuration = meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, {
+    this.provingJobDuration = this.meter.createHistogram(Metrics.PROVER_NODE_JOB_DURATION, {
       description: 'Duration of proving job',
       unit: 's',
       valueType: ValueType.DOUBLE,
     });
-    this.provingJobBlocks = meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, {
+    this.provingJobBlocks = this.meter.createGauge(Metrics.PROVER_NODE_JOB_BLOCKS, {
       description: 'Number of blocks in a proven epoch',
       valueType: ValueType.INT,
     });
-    this.provingJobTransactions = meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, {
+    this.provingJobTransactions = this.meter.createGauge(Metrics.PROVER_NODE_JOB_TRANSACTIONS, {
       description: 'Number of transactions in a proven epoch',
       valueType: ValueType.INT,
     });
+  }
+
+  public recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number) {
+    this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
+    this.provingJobDuration.record(totalTimeMs / 1000);
+    this.provingJobBlocks.record(Math.floor(numBlocks));
+    this.provingJobTransactions.record(Math.floor(numTxs));
+  }
+}
+
+export class ProverNodeRewardsMetrics {
+  private rewards: ObservableGauge;
+  private accumulatedRewards: UpDownCounter;
+  private prevEpoch = -1n;
+  private proofSubmissionWindow = 0n;
+
+  constructor(
+    private meter: Meter,
+    private coinbase: EthAddress,
+    private rollup: RollupContract,
+    private logger = createLogger('prover-node:publisher:metrics'),
+  ) {
+    this.rewards = this.meter.createObservableGauge(Metrics.PROVER_NODE_REWARDS_PER_EPOCH, {
+      valueType: ValueType.DOUBLE,
+      description: 'The rewards earned',
+    });
+
+    this.accumulatedRewards = this.meter.createUpDownCounter(Metrics.PROVER_NODE_REWARDS_TOTAL, {
+      valueType: ValueType.DOUBLE,
+      description: 'The rewards earned (total)',
+    });
+  }
+
+  public async start() {
+    this.proofSubmissionWindow = await this.rollup.getProofSubmissionWindow();
+    this.meter.addBatchObservableCallback(this.observe, [this.rewards]);
+  }
+
+  public stop() {
+    this.meter.removeBatchObservableCallback(this.observe, [this.rewards]);
+  }
+
+  private observe = async (observer: BatchObservableResult): Promise<void> => {
+    const slot = await this.rollup.getSlotNumber();
+
+    // look at the prev epoch so that we get an accurate value, after proof submission window has closed
+    if (slot > this.proofSubmissionWindow) {
+      const closedEpoch = await this.rollup.getEpochNumberForSlotNumber(slot - this.proofSubmissionWindow);
+      const rewards = await this.rollup.getSpecificProverRewardsForEpoch(closedEpoch, this.coinbase);
+
+      const fmt = parseFloat(formatUnits(rewards, 18));
+
+      observer.observe(this.rewards, fmt, {
+        [Attributes.COINBASE]: this.coinbase.toString(),
+      });
+
+      // only accumulate once per epoch
+      if (closedEpoch > this.prevEpoch) {
+        this.prevEpoch = closedEpoch;
+        this.accumulatedRewards.add(fmt, {
+          [Attributes.COINBASE]: this.coinbase.toString(),
+        });
+      }
+    }
+  };
+}
 
-    this.gasPrice = meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE, {
+export class ProverNodePublisherMetrics {
+  gasPrice: Histogram;
+  txCount: UpDownCounter;
+  txDuration: Histogram;
+  txGas: Histogram;
+  txCalldataSize: Histogram;
+  txCalldataGas: Histogram;
+  txBlobDataGasUsed: Histogram;
+  txBlobDataGasCost: Histogram;
+  txTotalFee: Histogram;
+
+  private senderBalance: Gauge;
+  private meter: Meter;
+
+  constructor(
+    public readonly client: TelemetryClient,
+    name = 'ProverNode',
+    private logger = createLogger('prover-node:publisher:metrics'),
+  ) {
+    this.meter = client.getMeter(name);
+
+    this.gasPrice = this.meter.createHistogram(Metrics.L1_PUBLISHER_GAS_PRICE, {
       description: 'The gas price used for transactions',
       unit: 'gwei',
       valueType: ValueType.DOUBLE,
     });
 
-    this.txCount = meter.createUpDownCounter(Metrics.L1_PUBLISHER_TX_COUNT, {
+    this.txCount = this.meter.createUpDownCounter(Metrics.L1_PUBLISHER_TX_COUNT, {
       description: 'The number of transactions processed',
     });
 
-    this.txDuration = meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION, {
+    this.txDuration = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_DURATION, {
       description: 'The duration of transaction processing',
       unit: 'ms',
       valueType: ValueType.INT,
     });
 
-    this.txGas = meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS, {
+    this.txGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_GAS, {
       description: 'The gas consumed by transactions',
       unit: 'gas',
       valueType: ValueType.INT,
     });
 
-    this.txCalldataSize = meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE, {
+    this.txCalldataSize = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_SIZE, {
       description: 'The size of the calldata in transactions',
       unit: 'By',
       valueType: ValueType.INT,
     });
 
-    this.txCalldataGas = meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS, {
+    this.txCalldataGas = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_CALLDATA_GAS, {
       description: 'The gas consumed by the calldata in transactions',
       unit: 'gas',
       valueType: ValueType.INT,
     });
 
-    this.txBlobDataGasUsed = meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED, {
+    this.txBlobDataGasUsed = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_USED, {
       description: 'The amount of blob gas used in transactions',
       unit: 'gas',
       valueType: ValueType.INT,
     });
 
-    this.txBlobDataGasCost = meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST, {
+    this.txBlobDataGasCost = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_BLOBDATA_GAS_COST, {
       description: 'The gas cost of blobs in transactions',
       unit: 'gwei',
       valueType: ValueType.INT,
     });
 
-    this.txTotalFee = meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE, {
+    this.txTotalFee = this.meter.createHistogram(Metrics.L1_PUBLISHER_TX_TOTAL_FEE, {
       description: 'How much L1 tx costs',
       unit: 'gwei',
       valueType: ValueType.DOUBLE,
@@ -112,7 +192,7 @@ export class ProverNodeMetrics {
       },
     });
 
-    this.senderBalance = meter.createGauge(Metrics.L1_PUBLISHER_BALANCE, {
+    this.senderBalance = this.meter.createGauge(Metrics.L1_PUBLISHER_BALANCE, {
       unit: 'eth',
       description: 'The balance of the sender address',
       valueType: ValueType.DOUBLE,
@@ -130,13 +210,6 @@ export class ProverNodeMetrics {
     this.recordTx(durationMs, stats);
   }
 
-  public recordProvingJob(executionTimeMs: number, totalTimeMs: number, numBlocks: number, numTxs: number) {
-    this.proverEpochExecutionDuration.record(Math.ceil(executionTimeMs));
-    this.provingJobDuration.record(totalTimeMs / 1000);
-    this.provingJobBlocks.record(Math.floor(numBlocks));
-    this.provingJobTransactions.record(Math.floor(numTxs));
-  }
-
   public recordSenderBalance(wei: bigint, senderAddress: string) {
     const eth = parseFloat(formatEther(wei, 'wei'));
     this.senderBalance.record(eth, {

yarn-project/prover-node/src/prover-node-publisher.ts

@@ -17,7 +17,7 @@ import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-clien
 
 import { type Hex, type TransactionReceipt, encodeFunctionData } from 'viem';
 
-import { ProverNodeMetrics } from './metrics.js';
+import { ProverNodePublisherMetrics } from './metrics.js';
 
 /**
  * Stats for a sent transaction.
@@ -40,7 +40,7 @@ export class ProverNodePublisher {
   private interruptibleSleep = new InterruptibleSleep();
   private sleepTimeMs: number;
   private interrupted = false;
-  private metrics: ProverNodeMetrics;
+  private metrics: ProverNodePublisherMetrics;
 
   protected log = createLogger('prover-node:l1-tx-publisher');
 
@@ -60,12 +60,16 @@ export class ProverNodePublisher {
 
     const telemetry = deps.telemetry ?? getTelemetryClient();
 
-    this.metrics = new ProverNodeMetrics(telemetry, 'ProverNode');
+    this.metrics = new ProverNodePublisherMetrics(telemetry, 'ProverNode');
 
     this.rollupContract = deps.rollupContract;
     this.l1TxUtils = deps.l1TxUtils;
   }
 
+  public getRollupContract() {
+    return this.rollupContract;
+  }
+
   /**
    * Calling `interrupt` will cause any in progress call to `publishRollup` to return `false` asap.
    * Be warned, the call may return false even if the tx subsequently gets successfully mined.

yarn-project/prover-node/src/prover-node.test.ts

@@ -1,5 +1,6 @@
 import { timesParallel } from '@aztec/foundation/collection';
 import { EthAddress } from '@aztec/foundation/eth-address';
+import { Fr } from '@aztec/foundation/fields';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
 import { retryUntil } from '@aztec/foundation/retry';
 import { sleep } from '@aztec/foundation/sleep';
@@ -70,7 +71,9 @@ describe('prover-node', () => {
     );
 
   beforeEach(async () => {
-    prover = mock<EpochProverManager>();
+    prover = mock<EpochProverManager>({
+      getProverId: () => Fr.random(),
+    });
     publisher = mock<ProverNodePublisher>();
     l2BlockSource = mock<L2BlockSource>();
     l1ToL2MessageSource = mock<L1ToL2MessageSource>();