Implement retry in bridges equivocation loop (#11283)

Implement retry mechanism in bridges equivocation loop

---------

Co-authored-by: cmd[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Branislav Kontur <bkontur@gmail.com>

Author: 3 people

bridges/relays/equivocation/src/block_checker.rs

@@ -23,6 +23,7 @@ use bp_header_chain::{FinalityProof, FindEquivocations as FindEquivocationsT};
 use finality_relay::FinalityProofsBuf;
 use futures::future::{BoxFuture, FutureExt};
 use num_traits::Saturating;
+use std::time::Duration;
 
 /// First step in the block checking state machine.
 ///
@@ -217,6 +218,44 @@ impl<P: EquivocationDetectionPipeline> BlockChecker<P> {
 		Self::ReadSyncedHeaders(ReadSyncedHeaders { target_block_num })
 	}
 
+	pub fn run_with_retry<'a, SC: SourceClient<P>, TC: TargetClient<P>>(
+		self,
+		source_client: &'a mut SC,
+		target_client: &'a mut TC,
+		finality_proofs_buf: &'a mut FinalityProofsBuf<P>,
+		reporter: &'a mut EquivocationsReporter<P, SC>,
+		retry_params: (u32, Duration),
+	) -> BoxFuture<'a, Result<(), Self>> {
+		let (max_attempts, retry_tick) = retry_params;
+		async move {
+			let mut block_checker = self;
+			let mut retry_range = (0..max_attempts).peekable();
+			while let Some(_) = retry_range.next() {
+				block_checker = match block_checker
+					.run(source_client, target_client, finality_proofs_buf, reporter)
+					.await
+				{
+					Ok(_) => return Ok(()),
+					Err(err) => err,
+				};
+
+				// We don't need to sleep after the last attempt
+				if retry_range.peek().is_some() {
+					tracing::warn!(
+						target: "bridge",
+						source=%P::SOURCE_NAME,
+						target=%P::TARGET_NAME,
+						"Error running block checker. Retrying."
+					);
+					tokio::time::sleep(retry_tick).await;
+				}
+			}
+
+			Err(block_checker)
+		}
+		.boxed()
+	}
+
 	pub fn run<'a, SC: SourceClient<P>, TC: TargetClient<P>>(
 		self,
 		source_client: &'a mut SC,

bridges/relays/equivocation/src/equivocation_loop.rs

@@ -109,11 +109,12 @@ impl<P: EquivocationDetectionPipeline, SC: SourceClient<P>, TC: TargetClient<P>>
 				self.finality_proofs_buf.fill(&mut self.finality_proofs_stream);
 				let block_checker = BlockChecker::new(current_block_number);
 				let _ = block_checker
-					.run(
+					.run_with_retry(
 						&mut self.source_client,
 						&mut self.target_client,
 						&mut self.finality_proofs_buf,
 						&mut self.reporter,
+						(5, tick),
 					)
 					.await;
 				current_block_number = current_block_number.saturating_add(1.into());

prdoc/pr_11283.prdoc

@@ -0,0 +1,5 @@
+title: Implement retry in bridges equivocation loop
+doc:
+- audience: Node Dev
+  description: Implement retry mechanism in bridges equivocation loop
+crates: []