Increase kitchensink wait time to 60 seconds

crates/config/src/lib.rs

@@ -77,7 +77,7 @@ pub struct Arguments {
     /// This argument controls which private keys the nodes should have access to and be added to
     /// its wallet signers. With a value of N, private keys (0, N] will be added to the signer set
     /// of the node.
-    #[arg(long = "private-keys-count", default_value_t = 15_000)]
+    #[arg(long = "private-keys-count", default_value_t = 100_000)]
     pub private_keys_to_add: usize,
 
     /// The differential testing leader node implementation.

crates/core/src/driver/mod.rs

@@ -4,9 +4,10 @@ use std::collections::HashMap;
 use std::marker::PhantomData;
 use std::path::PathBuf;
 
+use alloy::eips::BlockNumberOrTag;
 use alloy::json_abi::JsonAbi;
 use alloy::network::{Ethereum, TransactionBuilder};
-use alloy::primitives::U256;
+use alloy::primitives::{BlockNumber, U256};
 use alloy::rpc::types::TransactionReceipt;
 use alloy::rpc::types::trace::geth::{
     CallFrame, GethDebugBuiltInTracerType, GethDebugTracerType, GethDebugTracingOptions, GethTrace,
@@ -21,6 +22,7 @@ use alloy::{
 };
 use anyhow::Context;
 use indexmap::IndexMap;
+use revive_dt_format::traits::ResolverApi;
 use semver::Version;
 
 use revive_dt_format::case::{Case, CaseIdx};
@@ -29,6 +31,7 @@ use revive_dt_format::metadata::{ContractInstance, ContractPathAndIdent};
 use revive_dt_format::{input::Input, metadata::Metadata};
 use revive_dt_node::Node;
 use revive_dt_node_interaction::EthereumNode;
+use tracing::Instrument;
 
 use crate::Platform;
 
@@ -84,7 +87,13 @@ where
             .handle_input_call_frame_tracing(&execution_receipt, node)
             .await?;
         self.handle_input_variable_assignment(input, &tracing_result)?;
-        self.handle_input_expectations(input, &execution_receipt, node, &tracing_result)
+        let resolver = BlockPinnedResolver::<'_, T> {
+            node,
+            block_number: execution_receipt
+                .block_number
+                .context("Transaction was not included in a block")?,
+        };
+        self.handle_input_expectations(input, &execution_receipt, &resolver, &tracing_result)
             .await?;
         self.handle_input_diff(case_idx, execution_receipt, node)
             .await
@@ -242,7 +251,7 @@ where
         &mut self,
         input: &Input,
         execution_receipt: &TransactionReceipt,
-        node: &T::Blockchain,
+        resolver: &impl ResolverApi,
         tracing_result: &CallFrame,
     ) -> anyhow::Result<()> {
         let span = tracing::info_span!("Handling input expectations");
@@ -279,7 +288,7 @@ where
         for expectation in expectations.iter() {
             self.handle_input_expectation_item(
                 execution_receipt,
-                node,
+                resolver,
                 expectation,
                 tracing_result,
             )
@@ -292,7 +301,7 @@ where
     async fn handle_input_expectation_item(
         &mut self,
         execution_receipt: &TransactionReceipt,
-        node: &T::Blockchain,
+        resolver: &impl ResolverApi,
         expectation: &ExpectedOutput,
         tracing_result: &CallFrame,
     ) -> anyhow::Result<()> {
@@ -304,7 +313,6 @@ where
 
         let deployed_contracts = &mut self.deployed_contracts;
         let variables = &mut self.variables;
-        let chain_state_provider = node;
 
         // Handling the receipt state assertion.
         let expected = !expectation.exception;
@@ -327,12 +335,7 @@ where
             let expected = expected_calldata;
             let actual = &tracing_result.output.as_ref().unwrap_or_default();
             if !expected
-                .is_equivalent(
-                    actual,
-                    deployed_contracts,
-                    &*variables,
-                    chain_state_provider,
-                )
+                .is_equivalent(actual, deployed_contracts, &*variables, resolver)
                 .await?
             {
                 tracing::error!(
@@ -358,14 +361,16 @@ where
             }
 
             // Handling the events assertion.
-            for (expected_event, actual_event) in
-                expected_events.iter().zip(execution_receipt.logs())
+            for (event_idx, (expected_event, actual_event)) in expected_events
+                .iter()
+                .zip(execution_receipt.logs())
+                .enumerate()
             {
                 // Handling the emitter assertion.
                 if let Some(ref expected_address) = expected_event.address {
                     let expected = Address::from_slice(
                         Calldata::new_compound([expected_address])
-                            .calldata(deployed_contracts, &*variables, node)
+                            .calldata(deployed_contracts, &*variables, resolver)
                             .await?
                             .get(12..32)
                             .expect("Can't fail"),
@@ -373,6 +378,7 @@ where
                     let actual = actual_event.address();
                     if actual != expected {
                         tracing::error!(
+                            event_idx,
                             %expected,
                             %actual,
                             "Event emitter assertion failed",
@@ -392,15 +398,11 @@ where
                 {
                     let expected = Calldata::new_compound([expected]);
                     if !expected
-                        .is_equivalent(
-                            &actual.0,
-                            deployed_contracts,
-                            &*variables,
-                            chain_state_provider,
-                        )
+                        .is_equivalent(&actual.0, deployed_contracts, &*variables, resolver)
                         .await?
                     {
                         tracing::error!(
+                            event_idx,
                             ?execution_receipt,
                             ?expected,
                             ?actual,
@@ -416,15 +418,11 @@ where
                 let expected = &expected_event.values;
                 let actual = &actual_event.data().data;
                 if !expected
-                    .is_equivalent(
-                        &actual.0,
-                        deployed_contracts,
-                        &*variables,
-                        chain_state_provider,
-                    )
+                    .is_equivalent(&actual.0, deployed_contracts, &*variables, resolver)
                     .await?
                 {
                     tracing::error!(
+                        event_idx,
                         ?execution_receipt,
                         ?expected,
                         ?actual,
@@ -649,15 +647,16 @@ where
         let mut inputs_executed = 0;
         for (input_idx, input) in self.case.inputs_iterator().enumerate() {
             let tracing_span = tracing::info_span!("Handling input", input_idx);
-            let _guard = tracing_span.enter();
 
             let (leader_receipt, _, leader_diff) = self
                 .leader_state
                 .handle_input(self.metadata, self.case_idx, &input, self.leader_node)
+                .instrument(tracing_span.clone())
                 .await?;
             let (follower_receipt, _, follower_diff) = self
                 .follower_state
                 .handle_input(self.metadata, self.case_idx, &input, self.follower_node)
+                .instrument(tracing_span)
                 .await?;
 
             if leader_diff == follower_diff {
@@ -680,3 +679,67 @@ where
         Ok(inputs_executed)
     }
 }
+
+pub struct BlockPinnedResolver<'a, T: Platform> {
+    block_number: BlockNumber,
+    node: &'a T::Blockchain,
+}
+
+impl<'a, T: Platform> ResolverApi for BlockPinnedResolver<'a, T> {
+    async fn chain_id(&self) -> anyhow::Result<alloy::primitives::ChainId> {
+        self.node.chain_id().await
+    }
+
+    async fn block_gas_limit(&self, number: BlockNumberOrTag) -> anyhow::Result<u128> {
+        self.node
+            .block_gas_limit(self.resolve_block_number_or_tag(number))
+            .await
+    }
+
+    async fn block_coinbase(&self, number: BlockNumberOrTag) -> anyhow::Result<Address> {
+        self.node
+            .block_coinbase(self.resolve_block_number_or_tag(number))
+            .await
+    }
+
+    async fn block_difficulty(&self, number: BlockNumberOrTag) -> anyhow::Result<U256> {
+        self.node
+            .block_difficulty(self.resolve_block_number_or_tag(number))
+            .await
+    }
+
+    async fn block_hash(
+        &self,
+        number: BlockNumberOrTag,
+    ) -> anyhow::Result<alloy::primitives::BlockHash> {
+        self.node
+            .block_hash(self.resolve_block_number_or_tag(number))
+            .await
+    }
+
+    async fn block_timestamp(
+        &self,
+        number: BlockNumberOrTag,
+    ) -> anyhow::Result<alloy::primitives::BlockTimestamp> {
+        self.node
+            .block_timestamp(self.resolve_block_number_or_tag(number))
+            .await
+    }
+
+    async fn last_block_number(&self) -> anyhow::Result<alloy::primitives::BlockNumber> {
+        Ok(self.block_number)
+    }
+}
+
+impl<'a, T: Platform> BlockPinnedResolver<'a, T> {
+    fn resolve_block_number_or_tag(&self, number: BlockNumberOrTag) -> BlockNumberOrTag {
+        match number {
+            BlockNumberOrTag::Latest => BlockNumberOrTag::Number(self.block_number),
+            n @ BlockNumberOrTag::Finalized
+            | n @ BlockNumberOrTag::Safe
+            | n @ BlockNumberOrTag::Earliest
+            | n @ BlockNumberOrTag::Pending
+            | n @ BlockNumberOrTag::Number(_) => n,
+        }
+    }
+}

crates/core/src/main.rs

@@ -150,6 +150,18 @@ where
                     })
             },
         )
+        .filter(
+            |(metadata_file_path, metadata, _, _, _)| match metadata.ignore {
+                Some(true) => {
+                    tracing::warn!(
+                        metadata_file_path = %metadata_file_path.display(),
+                        "Ignoring metadata file"
+                    );
+                    false
+                }
+                Some(false) | None => true,
+            },
+        )
         .collect::<Vec<_>>();
 
     let metadata_case_status = Arc::new(RwLock::new(test_cases.iter().fold(
@@ -169,6 +181,8 @@ where
             const RESET: &str = "\x1B[0m";
 
             let mut entries_to_delete = Vec::new();
+            let mut number_of_successes = 0;
+            let mut number_of_failures = 0;
             loop {
                 let metadata_case_status_read = metadata_case_status.read().await;
                 if metadata_case_status_read.is_empty() {
@@ -204,6 +218,15 @@ where
                         )
                     };
 
+                    number_of_successes += case_status
+                        .values()
+                        .filter(|value| value.is_some_and(|value| value))
+                        .count();
+                    number_of_failures += case_status
+                        .values()
+                        .filter(|value| value.is_some_and(|value| !value))
+                        .count();
+
                     let mut case_status = case_status
                         .iter()
                         .map(|((case_idx, case_name), case_status)| {
@@ -243,6 +266,10 @@ where
 
                 tokio::time::sleep(std::time::Duration::from_secs(3)).await;
             }
+
+            eprintln!(
+                "{GREEN}{number_of_successes}{RESET} cases succeeded, {RED}{number_of_failures}{RESET} cases failed"
+            );
         }
     };
 
@@ -289,7 +316,7 @@ where
                             .entry((metadata_file_path.clone(), solc_mode))
                             .or_default()
                             .insert((CaseIdx::new(case_idx), case.name.clone()), Some(false));
-                        tracing::info!(%error, "Execution failed")
+                        tracing::error!(%error, "Execution failed")
                     }
                 }
                 tracing::info!("Execution completed");
@@ -545,7 +572,14 @@ async fn get_or_build_contracts<'a, P: Platform>(
             None => {
                 tracing::debug!(?key, "Compiled contracts cache miss");
                 let compiled_contracts = Arc::new(
-                    compile_contracts::<P>(metadata, &mode, config, deployed_libraries).await?,
+                    compile_contracts::<P>(
+                        metadata,
+                        metadata_file_path,
+                        &mode,
+                        config,
+                        deployed_libraries,
+                    )
+                    .await?,
                 );
                 *compilation_artifact = Some(compiled_contracts.clone());
                 return Ok(compiled_contracts.clone());
@@ -561,14 +595,23 @@ async fn get_or_build_contracts<'a, P: Platform>(
         mutex
     };
     let mut compilation_artifact = mutex.lock().await;
-    let compiled_contracts =
-        Arc::new(compile_contracts::<P>(metadata, &mode, config, deployed_libraries).await?);
+    let compiled_contracts = Arc::new(
+        compile_contracts::<P>(
+            metadata,
+            metadata_file_path,
+            &mode,
+            config,
+            deployed_libraries,
+        )
+        .await?,
+    );
     *compilation_artifact = Some(compiled_contracts.clone());
     Ok(compiled_contracts.clone())
 }
 
 async fn compile_contracts<P: Platform>(
     metadata: &Metadata,
+    metadata_file_path: &Path,
     mode: &SolcMode,
     config: &Arguments,
     deployed_libraries: &HashMap<ContractInstance, (Address, JsonAbi)>,
@@ -578,6 +621,13 @@ async fn compile_contracts<P: Platform>(
         P::Compiler::get_compiler_executable(config, compiler_version_or_requirement).await?;
     let compiler_version = P::Compiler::new(compiler_path.clone()).version()?;
 
+    tracing::info!(
+        %compiler_version,
+        metadata_file_path = %metadata_file_path.display(),
+        mode = ?mode,
+        "Compiling contracts"
+    );
+
     let compiler = Compiler::<P::Compiler>::new()
         .with_allow_path(metadata.directory()?)
         .with_optimization(mode.solc_optimize());
@@ -592,11 +642,11 @@ async fn compile_contracts<P: Platform>(
             .expect("Impossible for library to not be found in contracts")
             .contract_ident;
 
-        // Note the following: we need to tell solc which files require the libraries to be
-        // linked into them. We do not have access to this information and therefore we choose
-        // an easier, yet more compute intensive route, of telling solc that all of the files
-        // need to link the library and it will only perform the linking for the files that do
-        // actually need the library.
+        // Note the following: we need to tell solc which files require the libraries to be linked
+        // into them. We do not have access to this information and therefore we choose an easier,
+        // yet more compute intensive route, of telling solc that all of the files need to link the
+        // library and it will only perform the linking for the files that do actually need the
+        // library.
         compiler = FilesWithExtensionIterator::new(metadata.directory()?)
             .with_allowed_extension("sol")
             .fold(compiler, |compiler, path| {
@@ -646,6 +696,7 @@ async fn compile_corpus(
                 TestingPlatform::Geth => {
                     let _ = compile_contracts::<Geth>(
                         &metadata.content,
+                        &metadata.path,
                         &mode,
                         config,
                         &Default::default(),
@@ -655,6 +706,7 @@ async fn compile_corpus(
                 TestingPlatform::Kitchensink => {
                     let _ = compile_contracts::<Geth>(
                         &metadata.content,
+                        &metadata.path,
                         &mode,
                         config,
                         &Default::default(),

crates/format/src/input.rs

@@ -202,13 +202,13 @@ impl Input {
         &'a self,
         deployed_contracts: &HashMap<ContractInstance, (Address, JsonAbi)>,
         variables: impl Into<Option<&'a HashMap<String, U256>>> + Clone,
-        chain_state_provider: &impl ResolverApi,
+        resolver: &impl ResolverApi,
     ) -> anyhow::Result<Bytes> {
         match self.method {
             Method::Deployer | Method::Fallback => {
                 let calldata = self
                     .calldata
-                    .calldata(deployed_contracts, variables, chain_state_provider)
+                    .calldata(deployed_contracts, variables, resolver)
                     .await?;
 
                 Ok(calldata.into())
@@ -256,12 +256,7 @@ impl Input {
                 let mut calldata = Vec::<u8>::with_capacity(4 + self.calldata.size_requirement());
                 calldata.extend(function.selector().0);
                 self.calldata
-                    .calldata_into_slice(
-                        &mut calldata,
-                        deployed_contracts,
-                        variables,
-                        chain_state_provider,
-                    )
+                    .calldata_into_slice(&mut calldata, deployed_contracts, variables, resolver)
                     .await?;
 
                 Ok(calldata.into())
@@ -274,10 +269,10 @@ impl Input {
         &'a self,
         deployed_contracts: &HashMap<ContractInstance, (Address, JsonAbi)>,
         variables: impl Into<Option<&'a HashMap<String, U256>>> + Clone,
-        chain_state_provider: &impl ResolverApi,
+        resolver: &impl ResolverApi,
     ) -> anyhow::Result<TransactionRequest> {
         let input_data = self
-            .encoded_input(deployed_contracts, variables, chain_state_provider)
+            .encoded_input(deployed_contracts, variables, resolver)
             .await?;
         let transaction_request = TransactionRequest::default().from(self.caller).value(
             self.value
@@ -360,16 +355,11 @@ impl Calldata {
         &'a self,
         deployed_contracts: &HashMap<ContractInstance, (Address, JsonAbi)>,
         variables: impl Into<Option<&'a HashMap<String, U256>>> + Clone,
-        chain_state_provider: &impl ResolverApi,
+        resolver: &impl ResolverApi,
     ) -> anyhow::Result<Vec<u8>> {
         let mut buffer = Vec::<u8>::with_capacity(self.size_requirement());
-        self.calldata_into_slice(
-            &mut buffer,
-            deployed_contracts,
-            variables,
-            chain_state_provider,
-        )
-        .await?;
+        self.calldata_into_slice(&mut buffer, deployed_contracts, variables, resolver)
+            .await?;
         Ok(buffer)
     }
 
@@ -378,7 +368,7 @@ impl Calldata {
         buffer: &mut Vec<u8>,
         deployed_contracts: &HashMap<ContractInstance, (Address, JsonAbi)>,
         variables: impl Into<Option<&'a HashMap<String, U256>>> + Clone,
-        chain_state_provider: &impl ResolverApi,
+        resolver: &impl ResolverApi,
     ) -> anyhow::Result<()> {
         match self {
             Calldata::Single(bytes) => {
@@ -387,7 +377,7 @@ impl Calldata {
             Calldata::Compound(items) => {
                 for (arg_idx, arg) in items.iter().enumerate() {
                     match arg
-                        .resolve(deployed_contracts, variables.clone(), chain_state_provider)
+                        .resolve(deployed_contracts, variables.clone(), resolver)
                         .await
                     {
                         Ok(resolved) => {
@@ -417,7 +407,7 @@ impl Calldata {
         other: &[u8],
         deployed_contracts: &HashMap<ContractInstance, (Address, JsonAbi)>,
         variables: impl Into<Option<&'a HashMap<String, U256>>> + Clone,
-        chain_state_provider: &impl ResolverApi,
+        resolver: &impl ResolverApi,
     ) -> anyhow::Result<bool> {
         match self {
             Calldata::Single(calldata) => Ok(calldata == other),
@@ -440,7 +430,7 @@ impl Calldata {
                     };
 
                     let this = this
-                        .resolve(deployed_contracts, variables.clone(), chain_state_provider)
+                        .resolve(deployed_contracts, variables.clone(), resolver)
                         .await?;
                     let other = U256::from_be_slice(&other);
                     if this != other {
@@ -458,13 +448,13 @@ impl CalldataItem {
         &'a self,
         deployed_contracts: &HashMap<ContractInstance, (Address, JsonAbi)>,
         variables: impl Into<Option<&'a HashMap<String, U256>>> + Clone,
-        chain_state_provider: &impl ResolverApi,
+        resolver: &impl ResolverApi,
     ) -> anyhow::Result<U256> {
         let mut stack = Vec::<CalldataToken<U256>>::new();
 
         for token in self
             .calldata_tokens()
-            .map(|token| token.resolve(deployed_contracts, variables.clone(), chain_state_provider))
+            .map(|token| token.resolve(deployed_contracts, variables.clone(), resolver))
         {
             let token = token.await?;
             let new_token = match token {
@@ -569,7 +559,7 @@ impl<T: AsRef<str>> CalldataToken<T> {
         self,
         deployed_contracts: &HashMap<ContractInstance, (Address, JsonAbi)>,
         variables: impl Into<Option<&'a HashMap<String, U256>>> + Clone,
-        chain_state_provider: &impl ResolverApi,
+        resolver: &impl ResolverApi,
     ) -> anyhow::Result<CalldataToken<U256>> {
         match self {
             Self::Item(item) => {
@@ -599,22 +589,17 @@ impl<T: AsRef<str>> CalldataToken<T> {
                         anyhow::anyhow!("Invalid hexadecimal literal: {}", error)
                     })?)
                 } else if item == Self::CHAIN_VARIABLE {
-                    let chain_id = chain_state_provider.chain_id().await?;
+                    let chain_id = resolver.chain_id().await?;
                     Ok(U256::from(chain_id))
                 } else if item == Self::GAS_LIMIT_VARIABLE {
-                    let gas_limit = chain_state_provider
-                        .block_gas_limit(BlockNumberOrTag::Latest)
-                        .await?;
+                    let gas_limit = resolver.block_gas_limit(BlockNumberOrTag::Latest).await?;
                     Ok(U256::from(gas_limit))
                 } else if item == Self::COINBASE_VARIABLE {
-                    let coinbase = chain_state_provider
-                        .block_coinbase(BlockNumberOrTag::Latest)
-                        .await?;
+                    let coinbase = resolver.block_coinbase(BlockNumberOrTag::Latest).await?;
                     Ok(U256::from_be_slice(coinbase.as_ref()))
                 } else if item == Self::DIFFICULTY_VARIABLE {
-                    let block_difficulty = chain_state_provider
-                        .block_difficulty(BlockNumberOrTag::Latest)
-                        .await?;
+                    let block_difficulty =
+                        resolver.block_difficulty(BlockNumberOrTag::Latest).await?;
                     Ok(block_difficulty)
                 } else if item.starts_with(Self::BLOCK_HASH_VARIABLE_PREFIX) {
                     let offset: u64 = item
@@ -623,21 +608,17 @@ impl<T: AsRef<str>> CalldataToken<T> {
                         .and_then(|value| value.parse().ok())
                         .unwrap_or_default();
 
-                    let current_block_number = chain_state_provider.last_block_number().await?;
+                    let current_block_number = resolver.last_block_number().await?;
                     let desired_block_number = current_block_number - offset;
 
-                    let block_hash = chain_state_provider
-                        .block_hash(desired_block_number.into())
-                        .await?;
+                    let block_hash = resolver.block_hash(desired_block_number.into()).await?;
 
                     Ok(U256::from_be_bytes(block_hash.0))
                 } else if item == Self::BLOCK_NUMBER_VARIABLE {
-                    let current_block_number = chain_state_provider.last_block_number().await?;
+                    let current_block_number = resolver.last_block_number().await?;
                     Ok(U256::from(current_block_number))
                 } else if item == Self::BLOCK_TIMESTAMP_VARIABLE {
-                    let timestamp = chain_state_provider
-                        .block_timestamp(BlockNumberOrTag::Latest)
-                        .await?;
+                    let timestamp = resolver.block_timestamp(BlockNumberOrTag::Latest).await?;
                     Ok(U256::from(timestamp))
                 } else if let Some(variable_name) = item.strip_prefix(Self::VARIABLE_PREFIX) {
                     let Some(variables) = variables.into() else {
@@ -882,10 +863,10 @@ mod tests {
     async fn resolve_calldata_item(
         input: &str,
         deployed_contracts: &HashMap<ContractInstance, (Address, JsonAbi)>,
-        chain_state_provider: &impl ResolverApi,
+        resolver: &impl ResolverApi,
     ) -> anyhow::Result<U256> {
         CalldataItem::new(input)
-            .resolve(deployed_contracts, None, chain_state_provider)
+            .resolve(deployed_contracts, None, resolver)
             .await
     }
 

crates/node/src/kitchensink.rs

@@ -209,7 +209,7 @@ impl KitchensinkNode {
         if let Err(error) = Self::wait_ready(
             self.kitchensink_stderr_log_file_path().as_path(),
             Self::SUBSTRATE_READY_MARKER,
-            Duration::from_secs(30),
+            Duration::from_secs(60),
         ) {
             tracing::error!(
                 ?error,
@@ -238,7 +238,7 @@ impl KitchensinkNode {
         if let Err(error) = Self::wait_ready(
             self.proxy_stderr_log_file_path().as_path(),
             Self::ETH_PROXY_READY_MARKER,
-            Duration::from_secs(30),
+            Duration::from_secs(60),
         ) {
             tracing::error!(?error, "Failed to start proxy, shutting down gracefully");
             self.shutdown()?;