diff --git a/Cargo.lock b/Cargo.lock index 557f23b..501d257 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1644,6 +1644,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -2399,6 +2408,20 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42012b0f064e01aa58b545fe3727f90f7dd4020f4a3ea735b50344965f5a57e9" +[[package]] +name = "generator" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d18470a76cb7f8ff746cf1f7470914f900252ec36bbc40b569d74b1258446827" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -3165,6 +3188,19 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "lru" version = "0.13.0" @@ -3247,6 +3283,25 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "moka" +version = "0.12.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9321642ca94a4282428e6ea4af8cc2ca4eac48ac7a6a4ea8f33f76d0ce70926" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "loom", + "parking_lot", + "portable-atomic", + "rustc_version 0.4.1", + "smallvec", + "tagptr", + "thiserror 1.0.69", + "uuid", +] + [[package]] name = "native-tls" version = "0.2.14" @@ -3646,6 +3701,12 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + [[package]] name = "potential_utf" version = "0.1.2" @@ -4029,6 +4090,8 @@ name = "revive-dt-common" version = "0.1.0" dependencies = [ "anyhow", + "moka", + "once_cell", "semver 1.0.26", "tokio", ] @@ -4424,6 +4487,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -5276,6 +5345,12 @@ dependencies = [ "libc", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tap" version = "1.0.1" @@ -5808,6 +5883,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be" +dependencies = [ + "getrandom 0.3.3", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -6069,6 +6155,28 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-link", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core", +] + [[package]] name = "windows-core" version = "0.61.2" @@ -6082,6 +6190,17 @@ dependencies = [ "windows-strings 0.4.2", ] +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + [[package]] name = "windows-implement" version = "0.60.0" @@ -6106,9 +6225,19 @@ dependencies = [ [[package]] name = "windows-link" -version = "0.1.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core", + "windows-link", +] [[package]] name = "windows-registry" @@ -6198,6 +6327,15 @@ dependencies = [ "windows_x86_64_msvc 0.53.0", ] +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" diff --git a/Cargo.toml b/Cargo.toml index 20948e7..34ae899 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ clap = { version = "4", features = ["derive"] } foundry-compilers-artifacts = { version = "0.18.0" } futures = { version = "0.3.31" } hex = "0.4.3" +moka = "0.12.10" reqwest = { version = "0.12.15", features = ["json"] } once_cell = "1.21" semver = { version = "1.0", features = ["serde"] } diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 516b1be..52d7d8a 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -10,5 +10,7 @@ rust-version.workspace = true [dependencies] anyhow = { workspace = true } +moka = { workspace = true, features = ["sync"] } +once_cell = { workspace = true } semver = { workspace = true } tokio = { workspace = true, default-features = false, features = ["time"] } diff --git a/crates/common/src/cached_fs/mod.rs b/crates/common/src/cached_fs/mod.rs new file mode 100644 index 0000000..17a1fa5 --- /dev/null +++ b/crates/common/src/cached_fs/mod.rs @@ -0,0 +1,49 @@ +//! This module implements a cached file system allowing for results to be stored in-memory rather +//! rather being queried from the file system again. + +use std::fs; +use std::io::{Error, Result}; +use std::path::{Path, PathBuf}; + +use moka::sync::Cache; +use once_cell::sync::Lazy; + +pub fn read(path: impl AsRef) -> Result> { + static READ_CACHE: Lazy>> = Lazy::new(|| Cache::new(10_000)); + + let path = path.as_ref().canonicalize()?; + match READ_CACHE.get(path.as_path()) { + Some(content) => Ok(content), + None => { + let content = fs::read(path.as_path())?; + READ_CACHE.insert(path, content.clone()); + Ok(content) + } + } +} + +pub fn read_to_string(path: impl AsRef) -> Result { + let content = read(path)?; + String::from_utf8(content).map_err(|_| { + Error::new( + std::io::ErrorKind::InvalidData, + "The contents of the file are not valid UTF8", + ) + }) +} + +pub fn read_dir(path: impl AsRef) -> Result>>> { + static READ_DIR_CACHE: Lazy>> = Lazy::new(|| Cache::new(10_000)); + + let path = path.as_ref().canonicalize()?; + match READ_DIR_CACHE.get(path.as_path()) { + Some(entries) => Ok(Box::new(entries.into_iter().map(Ok)) as Box<_>), + None => { + let entries = fs::read_dir(path.as_path())? + .flat_map(|maybe_entry| maybe_entry.map(|entry| entry.path())) + .collect(); + READ_DIR_CACHE.insert(path.clone(), entries); + Ok(read_dir(path).unwrap()) + } + } +} diff --git a/crates/common/src/iterators/files_with_extension_iterator.rs b/crates/common/src/iterators/files_with_extension_iterator.rs index ac81103..93bd709 100644 --- a/crates/common/src/iterators/files_with_extension_iterator.rs +++ b/crates/common/src/iterators/files_with_extension_iterator.rs @@ -19,6 +19,11 @@ pub struct FilesWithExtensionIterator { /// this vector then they will be returned when the [`Iterator::next`] method is called. If not /// then we visit one of the next directories to visit. files_matching_allowed_extensions: Vec, + + /// This option controls if the the cached file system should be used or not. This could be + /// better for certain cases where the entries in the directories do not change and therefore + /// caching can be used. + use_cached_fs: bool, } impl FilesWithExtensionIterator { @@ -27,6 +32,7 @@ impl FilesWithExtensionIterator { allowed_extensions: Default::default(), directories_to_search: vec![root_directory.as_ref().to_path_buf()], files_matching_allowed_extensions: Default::default(), + use_cached_fs: Default::default(), } } @@ -37,6 +43,11 @@ impl FilesWithExtensionIterator { self.allowed_extensions.insert(allowed_extension.into()); self } + + pub fn with_use_cached_fs(mut self, use_cached_fs: bool) -> Self { + self.use_cached_fs = use_cached_fs; + self + } } impl Iterator for FilesWithExtensionIterator { @@ -49,16 +60,19 @@ impl Iterator for FilesWithExtensionIterator { let directory_to_search = self.directories_to_search.pop()?; - // Read all of the entries in the directory. If we failed to read this dir's entires then we - // elect to just ignore it and look in the next directory, we do that by calling the next - // method again on the iterator, which is an intentional decision that we made here instead - // of panicking. - let Ok(dir_entries) = std::fs::read_dir(directory_to_search) else { - return self.next(); + let iterator = if self.use_cached_fs { + let Ok(dir_entries) = crate::cached_fs::read_dir(directory_to_search.as_path()) else { + return self.next(); + }; + Box::new(dir_entries) as Box>> + } else { + let Ok(dir_entries) = std::fs::read_dir(directory_to_search) else { + return self.next(); + }; + Box::new(dir_entries.map(|maybe_entry| maybe_entry.map(|entry| entry.path()))) as Box<_> }; - for entry in dir_entries.flatten() { - let entry_path = entry.path(); + for entry_path in iterator.flatten() { if entry_path.is_dir() { self.directories_to_search.push(entry_path) } else if entry_path.is_file() diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 9649c36..60d360c 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -1,6 +1,7 @@ //! This crate provides common concepts, functionality, types, macros, and more that other crates in //! the workspace can benefit from. +pub mod cached_fs; pub mod fs; pub mod futures; pub mod iterators; diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 4337d35..dc00113 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -5,7 +5,6 @@ use std::{ collections::HashMap, - fs::read_to_string, hash::Hash, path::{Path, PathBuf}, }; @@ -16,6 +15,7 @@ use semver::Version; use serde::{Deserialize, Serialize}; use revive_common::EVMVersion; +use revive_dt_common::cached_fs::read_to_string; use revive_dt_common::types::VersionOrRequirement; use revive_dt_config::Arguments; diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs index 55ec7d3..8cad37a 100644 --- a/crates/core/src/main.rs +++ b/crates/core/src/main.rs @@ -714,6 +714,7 @@ async fn compile_contracts( // library. compiler = FilesWithExtensionIterator::new(metadata.directory()?) .with_allowed_extension("sol") + .with_use_cached_fs(true) .fold(compiler, |compiler, path| { compiler.with_library(&path, library_ident.as_str(), *library_address) }); diff --git a/crates/format/src/corpus.rs b/crates/format/src/corpus.rs index 43e07dc..1eec650 100644 --- a/crates/format/src/corpus.rs +++ b/crates/format/src/corpus.rs @@ -3,6 +3,7 @@ use std::{ path::{Path, PathBuf}, }; +use revive_dt_common::cached_fs::read_dir; use serde::{Deserialize, Serialize}; use crate::metadata::MetadataFile; @@ -54,7 +55,7 @@ impl Corpus { /// `path` is expected to be a directory. pub fn collect_metadata(path: &Path, tests: &mut Vec) { if path.is_dir() { - let dir_entry = match std::fs::read_dir(path) { + let dir_entry = match read_dir(path) { Ok(dir_entry) => dir_entry, Err(error) => { tracing::error!("failed to read dir '{}': {error}", path.display()); @@ -62,8 +63,8 @@ pub fn collect_metadata(path: &Path, tests: &mut Vec) { } }; - for entry in dir_entry { - let entry = match entry { + for path in dir_entry { + let path = match path { Ok(entry) => entry, Err(error) => { tracing::error!("error reading dir entry: {error}"); @@ -71,7 +72,6 @@ pub fn collect_metadata(path: &Path, tests: &mut Vec) { } }; - let path = entry.path(); if path.is_dir() { collect_metadata(&path, tests); continue; diff --git a/crates/format/src/metadata.rs b/crates/format/src/metadata.rs index 2a21c73..d5c55be 100644 --- a/crates/format/src/metadata.rs +++ b/crates/format/src/metadata.rs @@ -2,7 +2,7 @@ use std::{ cmp::Ordering, collections::BTreeMap, fmt::Display, - fs::{File, read_to_string}, + fs::File, ops::Deref, path::{Path, PathBuf}, str::FromStr, @@ -11,7 +11,9 @@ use std::{ use serde::{Deserialize, Serialize}; use revive_common::EVMVersion; -use revive_dt_common::{iterators::FilesWithExtensionIterator, macros::define_wrapper_type}; +use revive_dt_common::{ + cached_fs::read_to_string, iterators::FilesWithExtensionIterator, macros::define_wrapper_type, +}; use crate::{ case::Case, @@ -259,7 +261,9 @@ impl Metadata { Ok(Box::new(std::iter::once(metadata_file_path.clone()))) } else { Ok(Box::new( - FilesWithExtensionIterator::new(self.directory()?).with_allowed_extension("sol"), + FilesWithExtensionIterator::new(self.directory()?) + .with_allowed_extension("sol") + .with_use_cached_fs(true), )) } } diff --git a/crates/node/src/pool.rs b/crates/node/src/pool.rs index 8cb83cc..dfb71ab 100644 --- a/crates/node/src/pool.rs +++ b/crates/node/src/pool.rs @@ -1,11 +1,12 @@ //! This crate implements concurrent handling of testing node. use std::{ - fs::read_to_string, sync::atomic::{AtomicUsize, Ordering}, thread, }; +use revive_dt_common::cached_fs::read_to_string; + use anyhow::Context; use revive_dt_config::Arguments;