diff --git a/crates/common/src/fs/cached_file_system.rs b/crates/common/src/fs/cached_file_system.rs new file mode 100644 index 0000000..eb26852 --- /dev/null +++ b/crates/common/src/fs/cached_file_system.rs @@ -0,0 +1,38 @@ +//! Implements a cached file system that allows for files to be read once into memory and then when +//! they're requested to be read again they will be returned from the cache. + +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::{Arc, LazyLock}, +}; + +use anyhow::Result; +use tokio::sync::RwLock; + +#[allow(clippy::type_complexity)] +static CACHE: LazyLock>>>> = LazyLock::new(Default::default); + +pub struct CachedFileSystem; + +impl CachedFileSystem { + pub async fn read(path: impl AsRef) -> Result> { + let cache_read_lock = CACHE.read().await; + match cache_read_lock.get(path.as_ref()) { + Some(entry) => Ok(entry.clone()), + None => { + drop(cache_read_lock); + + let content = std::fs::read(&path)?; + let mut cache_write_lock = CACHE.write().await; + cache_write_lock.insert(path.as_ref().to_path_buf(), content.clone()); + Ok(content) + } + } + } + + pub async fn read_to_string(path: impl AsRef) -> Result { + let content = Self::read(path).await?; + String::from_utf8(content).map_err(Into::into) + } +} diff --git a/crates/common/src/fs/mod.rs b/crates/common/src/fs/mod.rs index f41c82e..ded01b2 100644 --- a/crates/common/src/fs/mod.rs +++ b/crates/common/src/fs/mod.rs @@ -1,3 +1,5 @@ +mod cached_file_system; mod clear_dir; +pub use cached_file_system::*; pub use clear_dir::*; diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 4337d35..201b7f7 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -5,7 +5,6 @@ use std::{ collections::HashMap, - fs::read_to_string, hash::Hash, path::{Path, PathBuf}, }; @@ -16,7 +15,7 @@ use semver::Version; use serde::{Deserialize, Serialize}; use revive_common::EVMVersion; -use revive_dt_common::types::VersionOrRequirement; +use revive_dt_common::{fs::CachedFileSystem, types::VersionOrRequirement}; use revive_dt_config::Arguments; pub mod revive_js; @@ -123,10 +122,11 @@ where self } - pub fn with_source(mut self, path: impl AsRef) -> anyhow::Result { - self.input - .sources - .insert(path.as_ref().to_path_buf(), read_to_string(path.as_ref())?); + pub async fn with_source(mut self, path: impl AsRef) -> anyhow::Result { + self.input.sources.insert( + path.as_ref().to_path_buf(), + CachedFileSystem::read_to_string(path.as_ref()).await?, + ); Ok(self) } diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs index 55ec7d3..7296ca5 100644 --- a/crates/core/src/main.rs +++ b/crates/core/src/main.rs @@ -67,7 +67,7 @@ fn main() -> anyhow::Result<()> { let args = init_cli()?; let body = async { - for (corpus, tests) in collect_corpora(&args)? { + for (corpus, tests) in collect_corpora(&args).await? { let span = Span::new(corpus, args.clone())?; match &args.compile_only { Some(platform) => compile_corpus(&args, &tests, platform, span).await, @@ -117,13 +117,13 @@ fn init_cli() -> anyhow::Result { Ok(args) } -fn collect_corpora(args: &Arguments) -> anyhow::Result>> { +async fn collect_corpora(args: &Arguments) -> anyhow::Result>> { let mut corpora = HashMap::new(); for path in &args.corpus { let corpus = Corpus::try_from_path(path)?; tracing::info!("found corpus: {}", path.display()); - let tests = corpus.enumerate_tests(); + let tests = corpus.enumerate_tests().await; tracing::info!("corpus '{}' contains {} tests", &corpus.name, tests.len()); corpora.insert(corpus, tests); } @@ -145,7 +145,7 @@ where let (report_tx, report_rx) = mpsc::unbounded_channel::<(Test, CaseResult)>(); let tests = prepare_tests::(metadata_files); - let driver_task = start_driver_task::(args, tests, span, report_tx)?; + let driver_task = start_driver_task::(args, tests, span, report_tx).await?; let status_reporter_task = start_reporter_task(report_rx); tokio::join!(status_reporter_task, driver_task); @@ -237,7 +237,7 @@ where }) } -fn start_driver_task( +async fn start_driver_task( args: &Arguments, tests: impl Iterator, span: Span, @@ -249,8 +249,8 @@ where L::Blockchain: revive_dt_node::Node + Send + Sync + 'static, F::Blockchain: revive_dt_node::Node + Send + Sync + 'static, { - let leader_nodes = Arc::new(NodePool::::new(args)?); - let follower_nodes = Arc::new(NodePool::::new(args)?); + let leader_nodes = Arc::new(NodePool::::new(args).await?); + let follower_nodes = Arc::new(NodePool::::new(args).await?); let compilation_cache = Arc::new(RwLock::new(HashMap::new())); let number_concurrent_tasks = args.number_of_concurrent_tasks(); @@ -693,12 +693,12 @@ async fn compile_contracts( "Compiling contracts" ); - let compiler = Compiler::::new() + let mut compiler = Compiler::::new() .with_allow_path(metadata.directory()?) .with_optimization(mode.solc_optimize()); - let mut compiler = metadata - .files_to_compile()? - .try_fold(compiler, |compiler, path| compiler.with_source(&path))?; + for path in metadata.files_to_compile()? { + compiler = compiler.with_source(path).await?; + } for (library_instance, (library_address, _)) in deployed_libraries.iter() { let library_ident = &metadata .contracts diff --git a/crates/format/src/corpus.rs b/crates/format/src/corpus.rs index 43e07dc..69bda27 100644 --- a/crates/format/src/corpus.rs +++ b/crates/format/src/corpus.rs @@ -39,9 +39,9 @@ impl Corpus { } /// Scan the corpus base directory and return all tests found. - pub fn enumerate_tests(&self) -> Vec { + pub async fn enumerate_tests(&self) -> Vec { let mut tests = Vec::new(); - collect_metadata(&self.path, &mut tests); + collect_metadata(&self.path, &mut tests).await; tests } } @@ -52,7 +52,7 @@ impl Corpus { /// Found tests are inserted into `tests`. /// /// `path` is expected to be a directory. -pub fn collect_metadata(path: &Path, tests: &mut Vec) { +pub async fn collect_metadata(path: &Path, tests: &mut Vec) { if path.is_dir() { let dir_entry = match std::fs::read_dir(path) { Ok(dir_entry) => dir_entry, @@ -73,12 +73,12 @@ pub fn collect_metadata(path: &Path, tests: &mut Vec) { let path = entry.path(); if path.is_dir() { - collect_metadata(&path, tests); + Box::pin(collect_metadata(&path, tests)).await; continue; } if path.is_file() { - if let Some(metadata) = MetadataFile::try_from_file(&path) { + if let Some(metadata) = MetadataFile::try_from_file(&path).await { tests.push(metadata) } } @@ -89,7 +89,7 @@ pub fn collect_metadata(path: &Path, tests: &mut Vec) { return; }; if extension.eq_ignore_ascii_case("sol") || extension.eq_ignore_ascii_case("json") { - if let Some(metadata) = MetadataFile::try_from_file(path) { + if let Some(metadata) = MetadataFile::try_from_file(path).await { tests.push(metadata) } } else { diff --git a/crates/format/src/metadata.rs b/crates/format/src/metadata.rs index 2a21c73..2d78f86 100644 --- a/crates/format/src/metadata.rs +++ b/crates/format/src/metadata.rs @@ -2,7 +2,6 @@ use std::{ cmp::Ordering, collections::BTreeMap, fmt::Display, - fs::{File, read_to_string}, ops::Deref, path::{Path, PathBuf}, str::FromStr, @@ -11,7 +10,9 @@ use std::{ use serde::{Deserialize, Serialize}; use revive_common::EVMVersion; -use revive_dt_common::{iterators::FilesWithExtensionIterator, macros::define_wrapper_type}; +use revive_dt_common::{ + fs::CachedFileSystem, iterators::FilesWithExtensionIterator, macros::define_wrapper_type, +}; use crate::{ case::Case, @@ -29,8 +30,8 @@ pub struct MetadataFile { } impl MetadataFile { - pub fn try_from_file(path: &Path) -> Option { - Metadata::try_from_file(path).map(|metadata| Self { + pub async fn try_from_file(path: &Path) -> Option { + Metadata::try_from_file(path).await.map(|metadata| Self { path: path.to_owned(), content: metadata, }) @@ -151,7 +152,7 @@ impl Metadata { /// /// # Panics /// Expects the supplied `path` to be a file. - pub fn try_from_file(path: &Path) -> Option { + pub async fn try_from_file(path: &Path) -> Option { assert!(path.is_file(), "not a file: {}", path.display()); let Some(file_extension) = path.extension() else { @@ -160,19 +161,20 @@ impl Metadata { }; if file_extension == METADATA_FILE_EXTENSION { - return Self::try_from_json(path); + return Self::try_from_json(path).await; } if file_extension == SOLIDITY_CASE_FILE_EXTENSION { - return Self::try_from_solidity(path); + return Self::try_from_solidity(path).await; } tracing::debug!("ignoring invalid corpus file: {}", path.display()); None } - fn try_from_json(path: &Path) -> Option { - let file = File::open(path) + async fn try_from_json(path: &Path) -> Option { + let content = CachedFileSystem::read(path) + .await .inspect_err(|error| { tracing::error!( "opening JSON test metadata file '{}' error: {error}", @@ -181,7 +183,7 @@ impl Metadata { }) .ok()?; - match serde_json::from_reader::<_, Metadata>(file) { + match serde_json::from_slice::(content.as_slice()) { Ok(mut metadata) => { metadata.file_path = Some(path.to_path_buf()); Some(metadata) @@ -196,8 +198,9 @@ impl Metadata { } } - fn try_from_solidity(path: &Path) -> Option { - let spec = read_to_string(path) + async fn try_from_solidity(path: &Path) -> Option { + let spec = CachedFileSystem::read_to_string(path) + .await .inspect_err(|error| { tracing::error!( "opening JSON test metadata file '{}' error: {error}", diff --git a/crates/node/src/pool.rs b/crates/node/src/pool.rs index 8cb83cc..c249e0d 100644 --- a/crates/node/src/pool.rs +++ b/crates/node/src/pool.rs @@ -1,12 +1,12 @@ //! This crate implements concurrent handling of testing node. use std::{ - fs::read_to_string, sync::atomic::{AtomicUsize, Ordering}, thread, }; use anyhow::Context; +use revive_dt_common::fs::CachedFileSystem; use revive_dt_config::Arguments; use crate::Node; @@ -23,12 +23,11 @@ where T: Node + Send + 'static, { /// Create a new Pool. This will start as many nodes as there are workers in `config`. - pub fn new(config: &Arguments) -> anyhow::Result { + pub async fn new(config: &Arguments) -> anyhow::Result { let nodes = config.number_of_nodes; - let genesis = read_to_string(&config.genesis_file).context(format!( - "can not read genesis file: {}", - config.genesis_file.display() - ))?; + let genesis = CachedFileSystem::read_to_string(&config.genesis_file) + .await + .context("Failed to read genesis file")?; let mut handles = Vec::with_capacity(nodes); for _ in 0..nodes {