chore: stop prover node swallowing start prover job errors (#13676)

Author: Maddiaa0

yarn-project/prover-node/src/monitors/epoch-monitor.test.ts

@@ -26,6 +26,9 @@ describe('EpochMonitor', () => {
     provenBlockNumber = 0;
 
     handler = mock<EpochMonitorHandler>();
+    // Default triggering proving jobs is successful
+    handler.handleEpochReadyToProve.mockResolvedValue(true);
+
     l2BlockSource = mock<L2BlockSource>({
       isEpochComplete(epochNumber) {
         return Promise.resolve(epochNumber <= lastEpochComplete);
@@ -87,6 +90,23 @@ describe('EpochMonitor', () => {
     expect(handler.handleEpochReadyToProve).not.toHaveBeenCalled();
   });
 
+  it('does not update the latest epoch number if proving was unable to start', async () => {
+    provenBlockNumber = 4;
+    blockToSlot[5] = 32n;
+    lastEpochComplete = 3n;
+
+    handler.handleEpochReadyToProve.mockResolvedValue(false);
+
+    await epochMonitor.work();
+    expect(handler.handleEpochReadyToProve).toHaveBeenCalledWith(3n);
+    expect(handler.handleEpochReadyToProve).toHaveBeenCalledTimes(1);
+
+    // It will be called again with the same epoch number
+    await epochMonitor.work();
+    expect(handler.handleEpochReadyToProve).toHaveBeenCalledWith(3n);
+    expect(handler.handleEpochReadyToProve).toHaveBeenCalledTimes(2);
+  });
+
   it('does not trigger epoch sync if epoch is already processed', async () => {
     provenBlockNumber = 4;
     blockToSlot[5] = 32n;

yarn-project/prover-node/src/monitors/epoch-monitor.ts

@@ -11,7 +11,7 @@ import {
 } from '@aztec/telemetry-client';
 
 export interface EpochMonitorHandler {
-  handleEpochReadyToProve(epochNumber: bigint): Promise<void>;
+  handleEpochReadyToProve(epochNumber: bigint): Promise<boolean>;
 }
 
 /**
@@ -71,6 +71,7 @@ export class EpochMonitor implements Traceable {
   @trackSpan('EpochMonitor.work')
   public async work() {
     const { epochToProve, blockNumber, slotNumber } = await this.getEpochNumberToProve();
+    this.log.debug(`Epoch to prove: ${epochToProve}`, { blockNumber, slotNumber });
     if (epochToProve === undefined) {
       this.log.trace(`Next block to prove ${blockNumber} not yet mined`, { blockNumber });
       return;
@@ -87,8 +88,9 @@ export class EpochMonitor implements Traceable {
     }
 
     this.log.debug(`Epoch ${epochToProve} is ready to be proven`);
-    await this.handler?.handleEpochReadyToProve(epochToProve);
-    this.latestEpochNumber = epochToProve;
+    if (await this.handler?.handleEpochReadyToProve(epochToProve)) {
+      this.latestEpochNumber = epochToProve;
+    }
   }
 
   private async getEpochNumberToProve() {

yarn-project/prover-node/src/prover-node.ts

@@ -123,8 +123,9 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
   /**
    * Handles an epoch being completed by starting a proof for it if there are no active jobs for it.
    * @param epochNumber - The epoch number that was just completed.
+   * @returns false if there is an error, true otherwise
    */
-  async handleEpochReadyToProve(epochNumber: bigint): Promise<void> {
+  async handleEpochReadyToProve(epochNumber: bigint): Promise<boolean> {
     try {
       this.log.debug(`Running jobs as ${epochNumber} is ready to prove`, {
         jobs: Array.from(this.jobs.values()).map(job => `${job.getEpochNumber()}:${job.getId()}`),
@@ -134,15 +135,17 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable
         this.log.warn(`Not starting proof for ${epochNumber} since there are active jobs for the epoch`, {
           activeJobs: activeJobs.map(job => job.uuid),
         });
-        return;
+        return true;
       }
       await this.startProof(epochNumber);
+      return true;
     } catch (err) {
       if (err instanceof EmptyEpochError) {
         this.log.info(`Not starting proof for ${epochNumber} since no blocks were found`);
       } else {
         this.log.error(`Error handling epoch completed`, err);
       }
+      return false;
     }
   }