Add a retry layer to all providers (#224)

* Add a `ReceiptRetryLayer` for providers

* Fix the retry layer

* Rename the retry layer

* Remove outdated polling function

* Remoe unneeded dependencies

crates/node/src/provider_utils/fallback_gas_filler.rs

@@ -1,5 +1,3 @@
-use std::{borrow::Cow, fmt::Display};
-
 use alloy::{
     eips::BlockNumberOrTag,
     network::{Network, TransactionBuilder},
@@ -111,28 +109,23 @@ where
                             },
                             state_overrides: Default::default(),
                             block_overrides: Default::default(),
+                            tx_index: Default::default(),
                         },
                     )
                     .await?
                     .try_into_call_frame()
                     .map_err(|err| {
-                        RpcError::LocalUsageError(
-                            FallbackGasFillerError::new(format!(
-                                "Expected a callframe trace, but got: {err:?}"
-                            ))
-                            .boxed(),
+                        RpcError::local_usage_str(
+                            format!("Expected a callframe trace, but got: {err:?}").as_str(),
                         )
                     })?;
 
                 let gas_used = u64::try_from(trace.gas_used).map_err(|_| {
-                    RpcError::LocalUsageError(
-                        FallbackGasFillerError::new(
-                            "Transaction trace returned a value of gas used that exceeds u64",
-                        )
-                        .boxed(),
+                    RpcError::local_usage_str(
+                        "Transaction trace returned a value of gas used that exceeds u64",
                     )
                 })?;
-                let gas_limit = gas_used.saturating_mul(120) / 100;
+                let gas_limit = gas_used.saturating_mul(2);
 
                 if let Some(gas_price) = tx.gas_price() {
                     return Ok(GasFillable::Legacy {
@@ -174,24 +167,3 @@ impl Default for FallbackGasFiller {
         Self::new()
     }
 }
-
-#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
-struct FallbackGasFillerError(Cow<'static, str>);
-
-impl FallbackGasFillerError {
-    pub fn new(string: impl Into<Cow<'static, str>>) -> Self {
-        Self(string.into())
-    }
-
-    pub fn boxed(self) -> Box<Self> {
-        Box::new(self)
-    }
-}
-
-impl Display for FallbackGasFillerError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        Display::fmt(&self.0, f)
-    }
-}
-
-impl std::error::Error for FallbackGasFillerError {}

crates/node/src/provider_utils/mod.rs

@@ -1,7 +1,9 @@
 mod concurrency_limiter;
 mod fallback_gas_filler;
 mod provider;
+mod receipt_retry_layer;
 
 pub use concurrency_limiter::*;
 pub use fallback_gas_filler::*;
 pub use provider::*;
+pub use receipt_retry_layer::*;

crates/node/src/provider_utils/provider.rs

@@ -1,18 +1,16 @@
-use std::{ops::ControlFlow, sync::LazyLock, time::Duration};
+use std::sync::LazyLock;
 
 use alloy::{
-    network::{Ethereum, Network, NetworkWallet, TransactionBuilder4844},
+    network::{Network, NetworkWallet, TransactionBuilder4844},
     providers::{
-        Identity, PendingTransactionBuilder, Provider, ProviderBuilder, RootProvider,
+        Identity, ProviderBuilder, RootProvider,
         fillers::{ChainIdFiller, FillProvider, JoinFill, NonceFiller, TxFiller, WalletFiller},
     },
     rpc::client::ClientBuilder,
 };
 use anyhow::{Context, Result};
-use revive_dt_common::futures::{PollingWaitBehavior, poll};
-use tracing::{Instrument, debug, info, info_span};
 
-use crate::provider_utils::{ConcurrencyLimiterLayer, FallbackGasFiller};
+use crate::provider_utils::{ConcurrencyLimiterLayer, FallbackGasFiller, RetryLayer};
 
 pub type ConcreteProvider<N, W> = FillProvider<
     JoinFill<
@@ -48,6 +46,7 @@ where
 
     let client = ClientBuilder::default()
         .layer(GLOBAL_CONCURRENCY_LIMITER_LAYER.clone())
+        .layer(RetryLayer::default())
         .connect(rpc_url)
         .await
         .context("Failed to construct the RPC client")?;
@@ -63,70 +62,3 @@ where
 
     Ok(provider)
 }
-
-pub async fn execute_transaction<N, W>(
-    provider: ConcreteProvider<N, W>,
-    transaction: N::TransactionRequest,
-) -> Result<N::ReceiptResponse>
-where
-    N: Network<
-            TransactionRequest: TransactionBuilder4844,
-            TxEnvelope = <Ethereum as Network>::TxEnvelope,
-        >,
-    W: NetworkWallet<N>,
-    Identity: TxFiller<N>,
-    FallbackGasFiller: TxFiller<N>,
-    ChainIdFiller: TxFiller<N>,
-    NonceFiller: TxFiller<N>,
-    WalletFiller<W>: TxFiller<N>,
-{
-    let sendable_transaction = provider
-        .fill(transaction)
-        .await
-        .context("Failed to fill transaction")?;
-
-    let transaction_envelope = sendable_transaction
-        .try_into_envelope()
-        .context("Failed to convert transaction into an envelope")?;
-    let tx_hash = *transaction_envelope.tx_hash();
-
-    let mut pending_transaction = match provider.send_tx_envelope(transaction_envelope).await {
-        Ok(pending_transaction) => pending_transaction,
-        Err(error) => {
-            let error_string = error.to_string();
-
-            if error_string.contains("Transaction Already Imported") {
-                PendingTransactionBuilder::<N>::new(provider.root().clone(), tx_hash)
-            } else {
-                return Err(error).context(format!("Failed to submit transaction {tx_hash}"));
-            }
-        }
-    };
-    debug!(%tx_hash, "Submitted Transaction");
-
-    pending_transaction.set_timeout(Some(Duration::from_secs(120)));
-    let tx_hash = pending_transaction.watch().await.context(format!(
-        "Transaction inclusion watching timeout for {tx_hash}"
-    ))?;
-
-    poll(
-        Duration::from_secs(60),
-        PollingWaitBehavior::Constant(Duration::from_secs(3)),
-        || {
-            let provider = provider.clone();
-
-            async move {
-                match provider.get_transaction_receipt(tx_hash).await {
-                    Ok(Some(receipt)) => {
-                        info!("Found the transaction receipt");
-                        Ok(ControlFlow::Break(receipt))
-                    }
-                    _ => Ok(ControlFlow::Continue(())),
-                }
-            }
-        },
-    )
-    .instrument(info_span!("Polling for receipt", %tx_hash))
-    .await
-    .context(format!("Polling for receipt failed for {tx_hash}"))
-}

crates/node/src/provider_utils/receipt_retry_layer.rs

@@ -0,0 +1,158 @@
+use std::time::Duration;
+
+use alloy::{
+    network::{AnyNetwork, Network},
+    rpc::json_rpc::{RequestPacket, ResponsePacket},
+    transports::{TransportError, TransportErrorKind, TransportFut},
+};
+use tokio::time::{interval, timeout};
+use tower::{Layer, Service};
+
+/// A layer that allows for automatic retries for getting the receipt.
+///
+/// There are certain cases where getting the receipt of a committed transaction might fail. In Geth
+/// this can happen if the transaction has been committed to the ledger but has not been indexed, in
+/// the substrate and revive stack it can also happen for other reasons.
+///
+/// Therefore, just because the first attempt to get the receipt (after transaction confirmation)
+/// has failed it doesn't mean that it will continue to fail. This layer can be added to any alloy
+/// provider to allow the provider to retry getting the receipt for some period of time before it
+/// considers that a timeout. It attempts to poll for the receipt for the `polling_duration` with an
+/// interval of `polling_interval` between each poll. If by the end of the `polling_duration` it was
+/// not able to get the receipt successfully then this is considered to be a timeout.
+///
+/// Additionally, this layer allows for retries for other rpc methods such as all tracing methods.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct RetryLayer {
+    /// The amount of time to keep polling for the receipt before considering it a timeout.
+    polling_duration: Duration,
+
+    /// The interval of time to wait between each poll for the receipt.
+    polling_interval: Duration,
+}
+
+impl RetryLayer {
+    pub fn new(polling_duration: Duration, polling_interval: Duration) -> Self {
+        Self {
+            polling_duration,
+            polling_interval,
+        }
+    }
+
+    pub fn with_polling_duration(mut self, polling_duration: Duration) -> Self {
+        self.polling_duration = polling_duration;
+        self
+    }
+
+    pub fn with_polling_interval(mut self, polling_interval: Duration) -> Self {
+        self.polling_interval = polling_interval;
+        self
+    }
+}
+
+impl Default for RetryLayer {
+    fn default() -> Self {
+        Self {
+            polling_duration: Duration::from_secs(90),
+            polling_interval: Duration::from_millis(500),
+        }
+    }
+}
+
+impl<S> Layer<S> for RetryLayer {
+    type Service = RetryService<S>;
+
+    fn layer(&self, inner: S) -> Self::Service {
+        RetryService {
+            service: inner,
+            polling_duration: self.polling_duration,
+            polling_interval: self.polling_interval,
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct RetryService<S> {
+    /// The internal service.
+    service: S,
+
+    /// The amount of time to keep polling for the receipt before considering it a timeout.
+    polling_duration: Duration,
+
+    /// The interval of time to wait between each poll for the receipt.
+    polling_interval: Duration,
+}
+
+impl<S> Service<RequestPacket> for RetryService<S>
+where
+    S: Service<RequestPacket, Future = TransportFut<'static>, Error = TransportError>
+        + Send
+        + 'static
+        + Clone,
+{
+    type Response = ResponsePacket;
+    type Error = TransportError;
+    type Future = TransportFut<'static>;
+
+    fn poll_ready(
+        &mut self,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        self.service.poll_ready(cx)
+    }
+
+    #[allow(clippy::nonminimal_bool)]
+    fn call(&mut self, req: RequestPacket) -> Self::Future {
+        type ReceiptOutput = <AnyNetwork as Network>::ReceiptResponse;
+
+        let mut service = self.service.clone();
+        let polling_interval = self.polling_interval;
+        let polling_duration = self.polling_duration;
+
+        Box::pin(async move {
+            let request = req.as_single().ok_or_else(|| {
+                TransportErrorKind::custom_str("Retry layer doesn't support batch requests")
+            })?;
+            let method = request.method();
+            let requires_retries = method == "eth_getTransactionReceipt"
+                || (method.contains("debug") && method.contains("trace"));
+
+            if !requires_retries {
+                return service.call(req).await;
+            }
+
+            timeout(polling_duration, async {
+                let mut interval = interval(polling_interval);
+
+                loop {
+                    interval.tick().await;
+
+                    let Ok(resp) = service.call(req.clone()).await else {
+                        continue;
+                    };
+                    let response = resp.as_single().expect("Can't fail");
+                    if response.is_error() {
+                        continue;
+                    }
+
+                    if method == "eth_getTransactionReceipt"
+                        && response
+                            .payload()
+                            .clone()
+                            .deserialize_success::<ReceiptOutput>()
+                            .ok()
+                            .and_then(|resp| resp.try_into_success().ok())
+                            .is_some()
+                        || method != "eth_getTransactionReceipt"
+                    {
+                        return resp;
+                    } else {
+                        continue;
+                    }
+                }
+            })
+            .await
+            .map_err(|_| TransportErrorKind::custom_str("Timeout when retrying request"))
+        })
+    }
+}