mirror of
https://github.com/pezkuwichain/revive-differential-tests.git
synced 2026-06-12 10:01:07 +00:00
Core Benchmarking Infra (#175)
* Implement a solution for the pre-fund account limit * Update the account pre-funding handling * Fix the lighthouse node tracing issue * refactor existing dt infra * Implement the platform driver * Wire up the cleaned up driver implementation * Implement the core benchmarking components * Remove some debug logging * Fix issues in the benchmarks driver * Implement a global concurrency limit on provider requests * Update the concurrency limit * Update the concurrency limit * Cleanups * Update the lighthouse ports * Ignore certain tests * Update the new geth test
This commit is contained in:
Omar
@@ -0,0 +1,375 @@
|
||||
//! A wrapper around the compiler which allows for caching of compilation artifacts so that they can
|
||||
//! be reused between runs.
|
||||
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
collections::HashMap,
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, LazyLock},
|
||||
};
|
||||
|
||||
use futures::FutureExt;
|
||||
use revive_dt_common::{iterators::FilesWithExtensionIterator, types::CompilerIdentifier};
|
||||
use revive_dt_compiler::{Compiler, CompilerOutput, Mode, SolidityCompiler};
|
||||
use revive_dt_core::Platform;
|
||||
use revive_dt_format::metadata::{ContractIdent, ContractInstance, Metadata};
|
||||
|
||||
use alloy::{hex::ToHexExt, json_abi::JsonAbi, primitives::Address};
|
||||
use anyhow::{Context as _, Error, Result};
|
||||
use revive_dt_report::ExecutionSpecificReporter;
|
||||
use semver::Version;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::{Mutex, RwLock, Semaphore};
|
||||
use tracing::{Instrument, debug, debug_span, instrument};
|
||||
|
||||
pub struct CachedCompiler<'a> {
|
||||
/// The cache that stores the compiled contracts.
|
||||
artifacts_cache: ArtifactsCache,
|
||||
|
||||
/// This is a mechanism that the cached compiler uses so that if multiple compilation requests
|
||||
/// come in for the same contract we never compile all of them and only compile it once and all
|
||||
/// other tasks that request this same compilation concurrently get the cached version.
|
||||
cache_key_lock: RwLock<HashMap<CacheKey<'a>, Arc<Mutex<()>>>>,
|
||||
}
|
||||
|
||||
impl<'a> CachedCompiler<'a> {
|
||||
pub async fn new(path: impl AsRef<Path>, invalidate_cache: bool) -> Result<Self> {
|
||||
let mut cache = ArtifactsCache::new(path);
|
||||
if invalidate_cache {
|
||||
cache = cache
|
||||
.with_invalidated_cache()
|
||||
.await
|
||||
.context("Failed to invalidate compilation cache directory")?;
|
||||
}
|
||||
Ok(Self {
|
||||
artifacts_cache: cache,
|
||||
cache_key_lock: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Compiles or gets the compilation artifacts from the cache.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
skip_all,
|
||||
fields(
|
||||
metadata_file_path = %metadata_file_path.display(),
|
||||
%mode,
|
||||
platform = %platform.platform_identifier()
|
||||
),
|
||||
err
|
||||
)]
|
||||
pub async fn compile_contracts(
|
||||
&self,
|
||||
metadata: &'a Metadata,
|
||||
metadata_file_path: &'a Path,
|
||||
mode: Cow<'a, Mode>,
|
||||
deployed_libraries: Option<&HashMap<ContractInstance, (ContractIdent, Address, JsonAbi)>>,
|
||||
compiler: &dyn SolidityCompiler,
|
||||
platform: &dyn Platform,
|
||||
reporter: &ExecutionSpecificReporter,
|
||||
) -> Result<CompilerOutput> {
|
||||
let cache_key = CacheKey {
|
||||
compiler_identifier: platform.compiler_identifier(),
|
||||
compiler_version: compiler.version().clone(),
|
||||
metadata_file_path,
|
||||
solc_mode: mode.clone(),
|
||||
};
|
||||
|
||||
let compilation_callback = || {
|
||||
async move {
|
||||
compile_contracts(
|
||||
metadata
|
||||
.directory()
|
||||
.context("Failed to get metadata directory while preparing compilation")?,
|
||||
metadata
|
||||
.files_to_compile()
|
||||
.context("Failed to enumerate files to compile from metadata")?,
|
||||
&mode,
|
||||
deployed_libraries,
|
||||
compiler,
|
||||
reporter,
|
||||
)
|
||||
.map(|compilation_result| compilation_result.map(CacheValue::new))
|
||||
.await
|
||||
}
|
||||
.instrument(debug_span!(
|
||||
"Running compilation for the cache key",
|
||||
cache_key.compiler_identifier = %cache_key.compiler_identifier,
|
||||
cache_key.compiler_version = %cache_key.compiler_version,
|
||||
cache_key.metadata_file_path = %cache_key.metadata_file_path.display(),
|
||||
cache_key.solc_mode = %cache_key.solc_mode,
|
||||
))
|
||||
};
|
||||
|
||||
let compiled_contracts = match deployed_libraries {
|
||||
// If deployed libraries have been specified then we will re-compile the contract as it
|
||||
// means that linking is required in this case.
|
||||
Some(_) => {
|
||||
debug!("Deployed libraries defined, recompilation must take place");
|
||||
debug!("Cache miss");
|
||||
compilation_callback()
|
||||
.await
|
||||
.context("Compilation callback for deployed libraries failed")?
|
||||
.compiler_output
|
||||
}
|
||||
// If no deployed libraries are specified then we can follow the cached flow and attempt
|
||||
// to lookup the compilation artifacts in the cache.
|
||||
None => {
|
||||
debug!("Deployed libraries undefined, attempting to make use of cache");
|
||||
|
||||
// Lock this specific cache key such that we do not get inconsistent state. We want
|
||||
// that when multiple cases come in asking for the compilation artifacts then they
|
||||
// don't all trigger a compilation if there's a cache miss. Hence, the lock here.
|
||||
let read_guard = self.cache_key_lock.read().await;
|
||||
let mutex = match read_guard.get(&cache_key).cloned() {
|
||||
Some(value) => {
|
||||
drop(read_guard);
|
||||
value
|
||||
}
|
||||
None => {
|
||||
drop(read_guard);
|
||||
self.cache_key_lock
|
||||
.write()
|
||||
.await
|
||||
.entry(cache_key.clone())
|
||||
.or_default()
|
||||
.clone()
|
||||
}
|
||||
};
|
||||
let _guard = mutex.lock().await;
|
||||
|
||||
match self.artifacts_cache.get(&cache_key).await {
|
||||
Some(cache_value) => {
|
||||
if deployed_libraries.is_some() {
|
||||
reporter
|
||||
.report_post_link_contracts_compilation_succeeded_event(
|
||||
compiler.version().clone(),
|
||||
compiler.path(),
|
||||
true,
|
||||
None,
|
||||
cache_value.compiler_output.clone(),
|
||||
)
|
||||
.expect("Can't happen");
|
||||
} else {
|
||||
reporter
|
||||
.report_pre_link_contracts_compilation_succeeded_event(
|
||||
compiler.version().clone(),
|
||||
compiler.path(),
|
||||
true,
|
||||
None,
|
||||
cache_value.compiler_output.clone(),
|
||||
)
|
||||
.expect("Can't happen");
|
||||
}
|
||||
cache_value.compiler_output
|
||||
}
|
||||
None => {
|
||||
let compiler_output = compilation_callback()
|
||||
.await
|
||||
.context("Compilation callback failed (cache miss path)")?
|
||||
.compiler_output;
|
||||
self.artifacts_cache
|
||||
.insert(
|
||||
&cache_key,
|
||||
&CacheValue {
|
||||
compiler_output: compiler_output.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.context(
|
||||
"Failed to write the cached value of the compilation artifacts",
|
||||
)?;
|
||||
compiler_output
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Ok(compiled_contracts)
|
||||
}
|
||||
}
|
||||
|
||||
async fn compile_contracts(
|
||||
metadata_directory: impl AsRef<Path>,
|
||||
mut files_to_compile: impl Iterator<Item = PathBuf>,
|
||||
mode: &Mode,
|
||||
deployed_libraries: Option<&HashMap<ContractInstance, (ContractIdent, Address, JsonAbi)>>,
|
||||
compiler: &dyn SolidityCompiler,
|
||||
reporter: &ExecutionSpecificReporter,
|
||||
) -> Result<CompilerOutput> {
|
||||
// Puts a limit on how many compilations we can perform at any given instance which helps us
|
||||
// with some of the errors we've been seeing with high concurrency on MacOS (we have not tried
|
||||
// it on Linux so we don't know if these issues also persist there or not.)
|
||||
static SPAWN_GATE: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(100));
|
||||
let _permit = SPAWN_GATE.acquire().await?;
|
||||
|
||||
let all_sources_in_dir = FilesWithExtensionIterator::new(metadata_directory.as_ref())
|
||||
.with_allowed_extension("sol")
|
||||
.with_use_cached_fs(true)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let compilation = Compiler::new()
|
||||
.with_allow_path(metadata_directory)
|
||||
// Handling the modes
|
||||
.with_optimization(mode.optimize_setting)
|
||||
.with_pipeline(mode.pipeline)
|
||||
// Adding the contract sources to the compiler.
|
||||
.try_then(|compiler| {
|
||||
files_to_compile.try_fold(compiler, |compiler, path| compiler.with_source(path))
|
||||
})?
|
||||
// Adding the deployed libraries to the compiler.
|
||||
.then(|compiler| {
|
||||
deployed_libraries
|
||||
.iter()
|
||||
.flat_map(|value| value.iter())
|
||||
.map(|(instance, (ident, address, abi))| (instance, ident, address, abi))
|
||||
.flat_map(|(_, ident, address, _)| {
|
||||
all_sources_in_dir
|
||||
.iter()
|
||||
.map(move |path| (ident, address, path))
|
||||
})
|
||||
.fold(compiler, |compiler, (ident, address, path)| {
|
||||
compiler.with_library(path, ident.as_str(), *address)
|
||||
})
|
||||
});
|
||||
|
||||
let input = compilation.input().clone();
|
||||
let output = compilation.try_build(compiler).await;
|
||||
|
||||
match (output.as_ref(), deployed_libraries.is_some()) {
|
||||
(Ok(output), true) => {
|
||||
reporter
|
||||
.report_post_link_contracts_compilation_succeeded_event(
|
||||
compiler.version().clone(),
|
||||
compiler.path(),
|
||||
false,
|
||||
input,
|
||||
output.clone(),
|
||||
)
|
||||
.expect("Can't happen");
|
||||
}
|
||||
(Ok(output), false) => {
|
||||
reporter
|
||||
.report_pre_link_contracts_compilation_succeeded_event(
|
||||
compiler.version().clone(),
|
||||
compiler.path(),
|
||||
false,
|
||||
input,
|
||||
output.clone(),
|
||||
)
|
||||
.expect("Can't happen");
|
||||
}
|
||||
(Err(err), true) => {
|
||||
reporter
|
||||
.report_post_link_contracts_compilation_failed_event(
|
||||
compiler.version().clone(),
|
||||
compiler.path().to_path_buf(),
|
||||
input,
|
||||
format!("{err:#}"),
|
||||
)
|
||||
.expect("Can't happen");
|
||||
}
|
||||
(Err(err), false) => {
|
||||
reporter
|
||||
.report_pre_link_contracts_compilation_failed_event(
|
||||
compiler.version().clone(),
|
||||
compiler.path().to_path_buf(),
|
||||
input,
|
||||
format!("{err:#}"),
|
||||
)
|
||||
.expect("Can't happen");
|
||||
}
|
||||
}
|
||||
|
||||
output
|
||||
}
|
||||
|
||||
struct ArtifactsCache {
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl ArtifactsCache {
|
||||
pub fn new(path: impl AsRef<Path>) -> Self {
|
||||
Self {
|
||||
path: path.as_ref().to_path_buf(),
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip_all, err)]
|
||||
pub async fn with_invalidated_cache(self) -> Result<Self> {
|
||||
cacache::clear(self.path.as_path())
|
||||
.await
|
||||
.map_err(Into::<Error>::into)
|
||||
.with_context(|| format!("Failed to clear cache at {}", self.path.display()))?;
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip_all, err)]
|
||||
pub async fn insert(&self, key: &CacheKey<'_>, value: &CacheValue) -> Result<()> {
|
||||
let key = bson::to_vec(key).context("Failed to serialize cache key (bson)")?;
|
||||
let value = bson::to_vec(value).context("Failed to serialize cache value (bson)")?;
|
||||
cacache::write(self.path.as_path(), key.encode_hex(), value)
|
||||
.await
|
||||
.with_context(|| {
|
||||
format!("Failed to write cache entry under {}", self.path.display())
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get(&self, key: &CacheKey<'_>) -> Option<CacheValue> {
|
||||
let key = bson::to_vec(key).ok()?;
|
||||
let value = cacache::read(self.path.as_path(), key.encode_hex())
|
||||
.await
|
||||
.ok()?;
|
||||
let value = bson::from_slice::<CacheValue>(&value).ok()?;
|
||||
Some(value)
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip_all, err)]
|
||||
pub async fn get_or_insert_with(
|
||||
&self,
|
||||
key: &CacheKey<'_>,
|
||||
callback: impl AsyncFnOnce() -> Result<CacheValue>,
|
||||
) -> Result<CacheValue> {
|
||||
match self.get(key).await {
|
||||
Some(value) => {
|
||||
debug!("Cache hit");
|
||||
Ok(value)
|
||||
}
|
||||
None => {
|
||||
debug!("Cache miss");
|
||||
let value = callback().await?;
|
||||
self.insert(key, &value).await?;
|
||||
Ok(value)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize)]
|
||||
struct CacheKey<'a> {
|
||||
/// The identifier of the used compiler.
|
||||
compiler_identifier: CompilerIdentifier,
|
||||
|
||||
/// The version of the compiler that was used to compile the artifacts.
|
||||
compiler_version: Version,
|
||||
|
||||
/// The path of the metadata file that the compilation artifacts are for.
|
||||
metadata_file_path: &'a Path,
|
||||
|
||||
/// The mode that the compilation artifacts where compiled with.
|
||||
solc_mode: Cow<'a, Mode>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
struct CacheValue {
|
||||
/// The compiler output from the compilation run.
|
||||
compiler_output: CompilerOutput,
|
||||
}
|
||||
|
||||
impl CacheValue {
|
||||
pub fn new(compiler_output: CompilerOutput) -> Self {
|
||||
Self { compiler_output }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
use revive_dt_config::CorpusConfiguration;
|
||||
use revive_dt_format::{corpus::Corpus, metadata::MetadataFile};
|
||||
use tracing::{info, info_span, instrument};
|
||||
|
||||
/// Given an object that implements [`AsRef<CorpusConfiguration>`], this function finds all of the
|
||||
/// corpus files and produces a map containing all of the [`MetadataFile`]s discovered.
|
||||
#[instrument(level = "debug", name = "Collecting Corpora", skip_all)]
|
||||
pub fn collect_metadata_files(
|
||||
context: impl AsRef<CorpusConfiguration>,
|
||||
) -> anyhow::Result<Vec<MetadataFile>> {
|
||||
let mut metadata_files = Vec::new();
|
||||
|
||||
let corpus_configuration = AsRef::<CorpusConfiguration>::as_ref(&context);
|
||||
for path in &corpus_configuration.paths {
|
||||
let span = info_span!("Processing corpus file", path = %path.display());
|
||||
let _guard = span.enter();
|
||||
|
||||
let corpus = Corpus::try_from_path(path)?;
|
||||
info!(
|
||||
name = corpus.name(),
|
||||
number_of_contained_paths = corpus.path_count(),
|
||||
"Deserialized corpus file"
|
||||
);
|
||||
metadata_files.extend(corpus.enumerate_tests());
|
||||
}
|
||||
|
||||
// There's a possibility that there are certain paths that all lead to the same metadata files
|
||||
// and therefore it's important that we sort them and then deduplicate them.
|
||||
metadata_files.sort_by(|a, b| a.metadata_file_path.cmp(&b.metadata_file_path));
|
||||
metadata_files.dedup_by(|a, b| a.metadata_file_path == b.metadata_file_path);
|
||||
|
||||
Ok(metadata_files)
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
mod cached_compiler;
|
||||
mod metadata;
|
||||
mod pool;
|
||||
mod test;
|
||||
|
||||
pub use cached_compiler::*;
|
||||
pub use metadata::*;
|
||||
pub use pool::*;
|
||||
pub use test::*;
|
||||
@@ -0,0 +1,59 @@
|
||||
//! This crate implements concurrent handling of testing node.
|
||||
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use anyhow::Context as _;
|
||||
use revive_dt_config::*;
|
||||
use revive_dt_core::Platform;
|
||||
use revive_dt_node_interaction::EthereumNode;
|
||||
|
||||
/// The node pool starts one or more [Node] which then can be accessed
|
||||
/// in a round robbin fashion.
|
||||
pub struct NodePool {
|
||||
next: AtomicUsize,
|
||||
nodes: Vec<Box<dyn EthereumNode + Send + Sync>>,
|
||||
}
|
||||
|
||||
impl NodePool {
|
||||
/// Create a new Pool. This will start as many nodes as there are workers in `config`.
|
||||
pub async fn new(context: Context, platform: &dyn Platform) -> anyhow::Result<Self> {
|
||||
let concurrency_configuration = AsRef::<ConcurrencyConfiguration>::as_ref(&context);
|
||||
let nodes = concurrency_configuration.number_of_nodes;
|
||||
|
||||
let mut handles = Vec::with_capacity(nodes);
|
||||
for _ in 0..nodes {
|
||||
let context = context.clone();
|
||||
handles.push(platform.new_node(context)?);
|
||||
}
|
||||
|
||||
let mut nodes = Vec::with_capacity(nodes);
|
||||
for handle in handles {
|
||||
nodes.push(
|
||||
handle
|
||||
.join()
|
||||
.map_err(|error| anyhow::anyhow!("failed to spawn node: {:?}", error))
|
||||
.context("Failed to join node spawn thread")?
|
||||
.context("Node failed to spawn")?,
|
||||
);
|
||||
}
|
||||
|
||||
let pre_transactions_tasks = nodes
|
||||
.iter_mut()
|
||||
.map(|node| node.pre_transactions())
|
||||
.collect::<Vec<_>>();
|
||||
futures::future::try_join_all(pre_transactions_tasks)
|
||||
.await
|
||||
.context("Failed to run the pre-transactions task")?;
|
||||
|
||||
Ok(Self {
|
||||
nodes,
|
||||
next: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Get a handle to the next node.
|
||||
pub fn round_robbin(&self) -> &dyn EthereumNode {
|
||||
let current = self.next.fetch_add(1, Ordering::SeqCst) % self.nodes.len();
|
||||
self.nodes.get(current).unwrap().as_ref()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::{borrow::Cow, path::Path};
|
||||
|
||||
use futures::{Stream, StreamExt, stream};
|
||||
use indexmap::{IndexMap, indexmap};
|
||||
use revive_dt_common::iterators::EitherIter;
|
||||
use revive_dt_common::types::PlatformIdentifier;
|
||||
use revive_dt_config::Context;
|
||||
use revive_dt_format::mode::ParsedMode;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use revive_dt_compiler::Mode;
|
||||
use revive_dt_compiler::SolidityCompiler;
|
||||
use revive_dt_format::{
|
||||
case::{Case, CaseIdx},
|
||||
metadata::MetadataFile,
|
||||
};
|
||||
use revive_dt_node_interaction::EthereumNode;
|
||||
use revive_dt_report::{ExecutionSpecificReporter, Reporter};
|
||||
use revive_dt_report::{TestSpecificReporter, TestSpecifier};
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
use crate::Platform;
|
||||
use crate::helpers::NodePool;
|
||||
|
||||
pub async fn create_test_definitions_stream<'a>(
|
||||
// This is only required for creating the compiler objects and is not used anywhere else in the
|
||||
// function.
|
||||
context: &Context,
|
||||
metadata_files: impl IntoIterator<Item = &'a MetadataFile>,
|
||||
platforms_and_nodes: &'a BTreeMap<PlatformIdentifier, (&dyn Platform, NodePool)>,
|
||||
reporter: Reporter,
|
||||
) -> impl Stream<Item = TestDefinition<'a>> {
|
||||
stream::iter(
|
||||
metadata_files
|
||||
.into_iter()
|
||||
// Flatten over the cases.
|
||||
.flat_map(|metadata_file| {
|
||||
metadata_file
|
||||
.cases
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(move |(case_idx, case)| (metadata_file, case_idx, case))
|
||||
})
|
||||
// Flatten over the modes, prefer the case modes over the metadata file modes.
|
||||
.flat_map(move |(metadata_file, case_idx, case)| {
|
||||
let reporter = reporter.clone();
|
||||
|
||||
let modes = case.modes.as_ref().or(metadata_file.modes.as_ref());
|
||||
let modes = match modes {
|
||||
Some(modes) => EitherIter::A(
|
||||
ParsedMode::many_to_modes(modes.iter()).map(Cow::<'static, _>::Owned),
|
||||
),
|
||||
None => EitherIter::B(Mode::all().map(Cow::<'static, _>::Borrowed)),
|
||||
};
|
||||
|
||||
modes.into_iter().map(move |mode| {
|
||||
(
|
||||
metadata_file,
|
||||
case_idx,
|
||||
case,
|
||||
mode.clone(),
|
||||
reporter.test_specific_reporter(Arc::new(TestSpecifier {
|
||||
solc_mode: mode.as_ref().clone(),
|
||||
metadata_file_path: metadata_file.metadata_file_path.clone(),
|
||||
case_idx: CaseIdx::new(case_idx),
|
||||
})),
|
||||
)
|
||||
})
|
||||
})
|
||||
// Inform the reporter of each one of the test cases that were discovered which we expect to
|
||||
// run.
|
||||
.inspect(|(_, _, _, _, reporter)| {
|
||||
reporter
|
||||
.report_test_case_discovery_event()
|
||||
.expect("Can't fail");
|
||||
}),
|
||||
)
|
||||
// Creating the Test Definition objects from all of the various objects we have and creating
|
||||
// their required dependencies (e.g., compiler).
|
||||
.filter_map(
|
||||
move |(metadata_file, case_idx, case, mode, reporter)| async move {
|
||||
let mut platforms = BTreeMap::new();
|
||||
for (platform, node_pool) in platforms_and_nodes.values() {
|
||||
let node = node_pool.round_robbin();
|
||||
let compiler = platform
|
||||
.new_compiler(context.clone(), mode.version.clone().map(Into::into))
|
||||
.await
|
||||
.inspect_err(|err| {
|
||||
error!(
|
||||
?err,
|
||||
platform_identifier = %platform.platform_identifier(),
|
||||
"Failed to instantiate the compiler"
|
||||
)
|
||||
})
|
||||
.ok()?;
|
||||
|
||||
reporter
|
||||
.report_node_assigned_event(
|
||||
node.id(),
|
||||
platform.platform_identifier(),
|
||||
node.connection_string(),
|
||||
)
|
||||
.expect("Can't fail");
|
||||
|
||||
let reporter =
|
||||
reporter.execution_specific_reporter(node.id(), platform.platform_identifier());
|
||||
|
||||
platforms.insert(
|
||||
platform.platform_identifier(),
|
||||
TestPlatformInformation {
|
||||
platform: *platform,
|
||||
node,
|
||||
compiler,
|
||||
reporter,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
Some(TestDefinition {
|
||||
/* Metadata file information */
|
||||
metadata: metadata_file,
|
||||
metadata_file_path: metadata_file.metadata_file_path.as_path(),
|
||||
|
||||
/* Mode Information */
|
||||
mode: mode.clone(),
|
||||
|
||||
/* Case Information */
|
||||
case_idx: CaseIdx::new(case_idx),
|
||||
case,
|
||||
|
||||
/* Platform and Node Assignment Information */
|
||||
platforms,
|
||||
|
||||
/* Reporter */
|
||||
reporter,
|
||||
})
|
||||
},
|
||||
)
|
||||
// Filter out the test cases which are incompatible or that can't run in the current setup.
|
||||
.filter_map(move |test| async move {
|
||||
match test.check_compatibility() {
|
||||
Ok(()) => Some(test),
|
||||
Err((reason, additional_information)) => {
|
||||
debug!(
|
||||
metadata_file_path = %test.metadata.metadata_file_path.display(),
|
||||
case_idx = %test.case_idx,
|
||||
mode = %test.mode,
|
||||
reason,
|
||||
additional_information =
|
||||
serde_json::to_string(&additional_information).unwrap(),
|
||||
"Ignoring Test Case"
|
||||
);
|
||||
test.reporter
|
||||
.report_test_ignored_event(
|
||||
reason.to_string(),
|
||||
additional_information
|
||||
.into_iter()
|
||||
.map(|(k, v)| (k.into(), v))
|
||||
.collect::<IndexMap<_, _>>(),
|
||||
)
|
||||
.expect("Can't fail");
|
||||
None
|
||||
}
|
||||
}
|
||||
})
|
||||
.inspect(|test| {
|
||||
info!(
|
||||
metadata_file_path = %test.metadata_file_path.display(),
|
||||
case_idx = %test.case_idx,
|
||||
mode = %test.mode,
|
||||
"Created a test case definition"
|
||||
);
|
||||
})
|
||||
}
|
||||
|
||||
/// This is a full description of a differential test to run alongside the full metadata file, the
|
||||
/// specific case to be tested, the platforms that the tests should run on, the specific nodes of
|
||||
/// these platforms that they should run on, the compilers to use, and everything else needed making
|
||||
/// it a complete description.
|
||||
pub struct TestDefinition<'a> {
|
||||
/* Metadata file information */
|
||||
pub metadata: &'a MetadataFile,
|
||||
pub metadata_file_path: &'a Path,
|
||||
|
||||
/* Mode Information */
|
||||
pub mode: Cow<'a, Mode>,
|
||||
|
||||
/* Case Information */
|
||||
pub case_idx: CaseIdx,
|
||||
pub case: &'a Case,
|
||||
|
||||
/* Platform and Node Assignment Information */
|
||||
pub platforms: BTreeMap<PlatformIdentifier, TestPlatformInformation<'a>>,
|
||||
|
||||
/* Reporter */
|
||||
pub reporter: TestSpecificReporter,
|
||||
}
|
||||
|
||||
impl<'a> TestDefinition<'a> {
|
||||
/// Checks if this test can be ran with the current configuration.
|
||||
pub fn check_compatibility(&self) -> TestCheckFunctionResult {
|
||||
self.check_metadata_file_ignored()?;
|
||||
self.check_case_file_ignored()?;
|
||||
self.check_target_compatibility()?;
|
||||
self.check_evm_version_compatibility()?;
|
||||
self.check_compiler_compatibility()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Checks if the metadata file is ignored or not.
|
||||
fn check_metadata_file_ignored(&self) -> TestCheckFunctionResult {
|
||||
if self.metadata.ignore.is_some_and(|ignore| ignore) {
|
||||
Err(("Metadata file is ignored.", indexmap! {}))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks if the case file is ignored or not.
|
||||
fn check_case_file_ignored(&self) -> TestCheckFunctionResult {
|
||||
if self.case.ignore.is_some_and(|ignore| ignore) {
|
||||
Err(("Case is ignored.", indexmap! {}))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks if the platforms all support the desired targets in the metadata file.
|
||||
fn check_target_compatibility(&self) -> TestCheckFunctionResult {
|
||||
let mut error_map = indexmap! {
|
||||
"test_desired_targets" => json!(self.metadata.targets.as_ref()),
|
||||
};
|
||||
let mut is_allowed = true;
|
||||
for (_, platform_information) in self.platforms.iter() {
|
||||
let is_allowed_for_platform = match self.metadata.targets.as_ref() {
|
||||
None => true,
|
||||
Some(required_vm_identifiers) => {
|
||||
required_vm_identifiers.contains(&platform_information.platform.vm_identifier())
|
||||
}
|
||||
};
|
||||
is_allowed &= is_allowed_for_platform;
|
||||
error_map.insert(
|
||||
platform_information.platform.platform_identifier().into(),
|
||||
json!(is_allowed_for_platform),
|
||||
);
|
||||
}
|
||||
|
||||
if is_allowed {
|
||||
Ok(())
|
||||
} else {
|
||||
Err((
|
||||
"One of the platforms do do not support the targets allowed by the test.",
|
||||
error_map,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
// Checks for the compatibility of the EVM version with the platforms specified.
|
||||
fn check_evm_version_compatibility(&self) -> TestCheckFunctionResult {
|
||||
let Some(evm_version_requirement) = self.metadata.required_evm_version else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let mut error_map = indexmap! {
|
||||
"test_desired_evm_version" => json!(self.metadata.required_evm_version),
|
||||
};
|
||||
let mut is_allowed = true;
|
||||
for (_, platform_information) in self.platforms.iter() {
|
||||
let is_allowed_for_platform =
|
||||
evm_version_requirement.matches(&platform_information.node.evm_version());
|
||||
is_allowed &= is_allowed_for_platform;
|
||||
error_map.insert(
|
||||
platform_information.platform.platform_identifier().into(),
|
||||
json!(is_allowed_for_platform),
|
||||
);
|
||||
}
|
||||
|
||||
if is_allowed {
|
||||
Ok(())
|
||||
} else {
|
||||
Err((
|
||||
"EVM version is incompatible for the platforms specified",
|
||||
error_map,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks if the platforms compilers support the mode that the test is for.
|
||||
fn check_compiler_compatibility(&self) -> TestCheckFunctionResult {
|
||||
let mut error_map = indexmap! {
|
||||
"test_desired_evm_version" => json!(self.metadata.required_evm_version),
|
||||
};
|
||||
let mut is_allowed = true;
|
||||
for (_, platform_information) in self.platforms.iter() {
|
||||
let is_allowed_for_platform = platform_information
|
||||
.compiler
|
||||
.supports_mode(self.mode.optimize_setting, self.mode.pipeline);
|
||||
is_allowed &= is_allowed_for_platform;
|
||||
error_map.insert(
|
||||
platform_information.platform.platform_identifier().into(),
|
||||
json!(is_allowed_for_platform),
|
||||
);
|
||||
}
|
||||
|
||||
if is_allowed {
|
||||
Ok(())
|
||||
} else {
|
||||
Err((
|
||||
"Compilers do not support this mode either for the provided platforms.",
|
||||
error_map,
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TestPlatformInformation<'a> {
|
||||
pub platform: &'a dyn Platform,
|
||||
pub node: &'a dyn EthereumNode,
|
||||
pub compiler: Box<dyn SolidityCompiler>,
|
||||
pub reporter: ExecutionSpecificReporter,
|
||||
}
|
||||
|
||||
type TestCheckFunctionResult = Result<(), (&'static str, IndexMap<&'static str, Value>)>;
|
||||
Reference in New Issue
Block a user