mirror of
https://github.com/pezkuwichain/revive-differential-tests.git
synced 2026-06-12 15:51:09 +00:00
Core Benchmarking Infra (#175)
* Implement a solution for the pre-fund account limit * Update the account pre-funding handling * Fix the lighthouse node tracing issue * refactor existing dt infra * Implement the platform driver * Wire up the cleaned up driver implementation * Implement the core benchmarking components * Remove some debug logging * Fix issues in the benchmarks driver * Implement a global concurrency limit on provider requests * Update the concurrency limit * Update the concurrency limit * Cleanups * Update the lighthouse ports * Ignore certain tests * Update the new geth test
This commit is contained in:
Omar
@@ -0,0 +1,325 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::{borrow::Cow, path::Path};
|
||||
|
||||
use futures::{Stream, StreamExt, stream};
|
||||
use indexmap::{IndexMap, indexmap};
|
||||
use revive_dt_common::iterators::EitherIter;
|
||||
use revive_dt_common::types::PlatformIdentifier;
|
||||
use revive_dt_config::Context;
|
||||
use revive_dt_format::mode::ParsedMode;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use revive_dt_compiler::Mode;
|
||||
use revive_dt_compiler::SolidityCompiler;
|
||||
use revive_dt_format::{
|
||||
case::{Case, CaseIdx},
|
||||
metadata::MetadataFile,
|
||||
};
|
||||
use revive_dt_node_interaction::EthereumNode;
|
||||
use revive_dt_report::{ExecutionSpecificReporter, Reporter};
|
||||
use revive_dt_report::{TestSpecificReporter, TestSpecifier};
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
use crate::Platform;
|
||||
use crate::helpers::NodePool;
|
||||
|
||||
pub async fn create_test_definitions_stream<'a>(
|
||||
// This is only required for creating the compiler objects and is not used anywhere else in the
|
||||
// function.
|
||||
context: &Context,
|
||||
metadata_files: impl IntoIterator<Item = &'a MetadataFile>,
|
||||
platforms_and_nodes: &'a BTreeMap<PlatformIdentifier, (&dyn Platform, NodePool)>,
|
||||
reporter: Reporter,
|
||||
) -> impl Stream<Item = TestDefinition<'a>> {
|
||||
stream::iter(
|
||||
metadata_files
|
||||
.into_iter()
|
||||
// Flatten over the cases.
|
||||
.flat_map(|metadata_file| {
|
||||
metadata_file
|
||||
.cases
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(move |(case_idx, case)| (metadata_file, case_idx, case))
|
||||
})
|
||||
// Flatten over the modes, prefer the case modes over the metadata file modes.
|
||||
.flat_map(move |(metadata_file, case_idx, case)| {
|
||||
let reporter = reporter.clone();
|
||||
|
||||
let modes = case.modes.as_ref().or(metadata_file.modes.as_ref());
|
||||
let modes = match modes {
|
||||
Some(modes) => EitherIter::A(
|
||||
ParsedMode::many_to_modes(modes.iter()).map(Cow::<'static, _>::Owned),
|
||||
),
|
||||
None => EitherIter::B(Mode::all().map(Cow::<'static, _>::Borrowed)),
|
||||
};
|
||||
|
||||
modes.into_iter().map(move |mode| {
|
||||
(
|
||||
metadata_file,
|
||||
case_idx,
|
||||
case,
|
||||
mode.clone(),
|
||||
reporter.test_specific_reporter(Arc::new(TestSpecifier {
|
||||
solc_mode: mode.as_ref().clone(),
|
||||
metadata_file_path: metadata_file.metadata_file_path.clone(),
|
||||
case_idx: CaseIdx::new(case_idx),
|
||||
})),
|
||||
)
|
||||
})
|
||||
})
|
||||
// Inform the reporter of each one of the test cases that were discovered which we expect to
|
||||
// run.
|
||||
.inspect(|(_, _, _, _, reporter)| {
|
||||
reporter
|
||||
.report_test_case_discovery_event()
|
||||
.expect("Can't fail");
|
||||
}),
|
||||
)
|
||||
// Creating the Test Definition objects from all of the various objects we have and creating
|
||||
// their required dependencies (e.g., compiler).
|
||||
.filter_map(
|
||||
move |(metadata_file, case_idx, case, mode, reporter)| async move {
|
||||
let mut platforms = BTreeMap::new();
|
||||
for (platform, node_pool) in platforms_and_nodes.values() {
|
||||
let node = node_pool.round_robbin();
|
||||
let compiler = platform
|
||||
.new_compiler(context.clone(), mode.version.clone().map(Into::into))
|
||||
.await
|
||||
.inspect_err(|err| {
|
||||
error!(
|
||||
?err,
|
||||
platform_identifier = %platform.platform_identifier(),
|
||||
"Failed to instantiate the compiler"
|
||||
)
|
||||
})
|
||||
.ok()?;
|
||||
|
||||
reporter
|
||||
.report_node_assigned_event(
|
||||
node.id(),
|
||||
platform.platform_identifier(),
|
||||
node.connection_string(),
|
||||
)
|
||||
.expect("Can't fail");
|
||||
|
||||
let reporter =
|
||||
reporter.execution_specific_reporter(node.id(), platform.platform_identifier());
|
||||
|
||||
platforms.insert(
|
||||
platform.platform_identifier(),
|
||||
TestPlatformInformation {
|
||||
platform: *platform,
|
||||
node,
|
||||
compiler,
|
||||
reporter,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
Some(TestDefinition {
|
||||
/* Metadata file information */
|
||||
metadata: metadata_file,
|
||||
metadata_file_path: metadata_file.metadata_file_path.as_path(),
|
||||
|
||||
/* Mode Information */
|
||||
mode: mode.clone(),
|
||||
|
||||
/* Case Information */
|
||||
case_idx: CaseIdx::new(case_idx),
|
||||
case,
|
||||
|
||||
/* Platform and Node Assignment Information */
|
||||
platforms,
|
||||
|
||||
/* Reporter */
|
||||
reporter,
|
||||
})
|
||||
},
|
||||
)
|
||||
// Filter out the test cases which are incompatible or that can't run in the current setup.
|
||||
.filter_map(move |test| async move {
|
||||
match test.check_compatibility() {
|
||||
Ok(()) => Some(test),
|
||||
Err((reason, additional_information)) => {
|
||||
debug!(
|
||||
metadata_file_path = %test.metadata.metadata_file_path.display(),
|
||||
case_idx = %test.case_idx,
|
||||
mode = %test.mode,
|
||||
reason,
|
||||
additional_information =
|
||||
serde_json::to_string(&additional_information).unwrap(),
|
||||
"Ignoring Test Case"
|
||||
);
|
||||
test.reporter
|
||||
.report_test_ignored_event(
|
||||
reason.to_string(),
|
||||
additional_information
|
||||
.into_iter()
|
||||
.map(|(k, v)| (k.into(), v))
|
||||
.collect::<IndexMap<_, _>>(),
|
||||
)
|
||||
.expect("Can't fail");
|
||||
None
|
||||
}
|
||||
}
|
||||
})
|
||||
.inspect(|test| {
|
||||
info!(
|
||||
metadata_file_path = %test.metadata_file_path.display(),
|
||||
case_idx = %test.case_idx,
|
||||
mode = %test.mode,
|
||||
"Created a test case definition"
|
||||
);
|
||||
})
|
||||
}
|
||||
|
||||
/// This is a full description of a differential test to run alongside the full metadata file, the
|
||||
/// specific case to be tested, the platforms that the tests should run on, the specific nodes of
|
||||
/// these platforms that they should run on, the compilers to use, and everything else needed making
|
||||
/// it a complete description.
|
||||
pub struct TestDefinition<'a> {
|
||||
/* Metadata file information */
|
||||
pub metadata: &'a MetadataFile,
|
||||
pub metadata_file_path: &'a Path,
|
||||
|
||||
/* Mode Information */
|
||||
pub mode: Cow<'a, Mode>,
|
||||
|
||||
/* Case Information */
|
||||
pub case_idx: CaseIdx,
|
||||
pub case: &'a Case,
|
||||
|
||||
/* Platform and Node Assignment Information */
|
||||
pub platforms: BTreeMap<PlatformIdentifier, TestPlatformInformation<'a>>,
|
||||
|
||||
/* Reporter */
|
||||
pub reporter: TestSpecificReporter,
|
||||
}
|
||||
|
||||
impl<'a> TestDefinition<'a> {
|
||||
/// Checks if this test can be ran with the current configuration.
|
||||
pub fn check_compatibility(&self) -> TestCheckFunctionResult {
|
||||
self.check_metadata_file_ignored()?;
|
||||
self.check_case_file_ignored()?;
|
||||
self.check_target_compatibility()?;
|
||||
self.check_evm_version_compatibility()?;
|
||||
self.check_compiler_compatibility()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Checks if the metadata file is ignored or not.
|
||||
fn check_metadata_file_ignored(&self) -> TestCheckFunctionResult {
|
||||
if self.metadata.ignore.is_some_and(|ignore| ignore) {
|
||||
Err(("Metadata file is ignored.", indexmap! {}))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks if the case file is ignored or not.
|
||||
fn check_case_file_ignored(&self) -> TestCheckFunctionResult {
|
||||
if self.case.ignore.is_some_and(|ignore| ignore) {
|
||||
Err(("Case is ignored.", indexmap! {}))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks if the platforms all support the desired targets in the metadata file.
|
||||
fn check_target_compatibility(&self) -> TestCheckFunctionResult {
|
||||
let mut error_map = indexmap! {
|
||||
"test_desired_targets" => json!(self.metadata.targets.as_ref()),
|
||||
};
|
||||
let mut is_allowed = true;
|
||||
for (_, platform_information) in self.platforms.iter() {
|
||||
let is_allowed_for_platform = match self.metadata.targets.as_ref() {
|
||||
None => true,
|
||||
Some(required_vm_identifiers) => {
|
||||
required_vm_identifiers.contains(&platform_information.platform.vm_identifier())
|
||||
}
|
||||
};
|
||||
is_allowed &= is_allowed_for_platform;
|
||||
error_map.insert(
|
||||
platform_information.platform.platform_identifier().into(),
|
||||
json!(is_allowed_for_platform),
|
||||
);
|
||||
}
|
||||
|
||||
if is_allowed {
|
||||
Ok(())
|
||||
} else {
|
||||
Err((
|
||||
"One of the platforms do do not support the targets allowed by the test.",
|
||||
error_map,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
// Checks for the compatibility of the EVM version with the platforms specified.
|
||||
fn check_evm_version_compatibility(&self) -> TestCheckFunctionResult {
|
||||
let Some(evm_version_requirement) = self.metadata.required_evm_version else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let mut error_map = indexmap! {
|
||||
"test_desired_evm_version" => json!(self.metadata.required_evm_version),
|
||||
};
|
||||
let mut is_allowed = true;
|
||||
for (_, platform_information) in self.platforms.iter() {
|
||||
let is_allowed_for_platform =
|
||||
evm_version_requirement.matches(&platform_information.node.evm_version());
|
||||
is_allowed &= is_allowed_for_platform;
|
||||
error_map.insert(
|
||||
platform_information.platform.platform_identifier().into(),
|
||||
json!(is_allowed_for_platform),
|
||||
);
|
||||
}
|
||||
|
||||
if is_allowed {
|
||||
Ok(())
|
||||
} else {
|
||||
Err((
|
||||
"EVM version is incompatible for the platforms specified",
|
||||
error_map,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks if the platforms compilers support the mode that the test is for.
|
||||
fn check_compiler_compatibility(&self) -> TestCheckFunctionResult {
|
||||
let mut error_map = indexmap! {
|
||||
"test_desired_evm_version" => json!(self.metadata.required_evm_version),
|
||||
};
|
||||
let mut is_allowed = true;
|
||||
for (_, platform_information) in self.platforms.iter() {
|
||||
let is_allowed_for_platform = platform_information
|
||||
.compiler
|
||||
.supports_mode(self.mode.optimize_setting, self.mode.pipeline);
|
||||
is_allowed &= is_allowed_for_platform;
|
||||
error_map.insert(
|
||||
platform_information.platform.platform_identifier().into(),
|
||||
json!(is_allowed_for_platform),
|
||||
);
|
||||
}
|
||||
|
||||
if is_allowed {
|
||||
Ok(())
|
||||
} else {
|
||||
Err((
|
||||
"Compilers do not support this mode either for the provided platforms.",
|
||||
error_map,
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TestPlatformInformation<'a> {
|
||||
pub platform: &'a dyn Platform,
|
||||
pub node: &'a dyn EthereumNode,
|
||||
pub compiler: Box<dyn SolidityCompiler>,
|
||||
pub reporter: ExecutionSpecificReporter,
|
||||
}
|
||||
|
||||
type TestCheckFunctionResult = Result<(), (&'static str, IndexMap<&'static str, Value>)>;
|
||||
Reference in New Issue
Block a user