diff --git a/CHANGELOG.md b/CHANGELOG.md index b3a2984..c63c6c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ This is a development pre-release. Supported `polkadot-sdk` rev: `unstable2507` +### Changed +- Removed the `revive-explorer` utility. + ### Fixed - OOB access in `calldataload` and `calldatacopy` should always produce zero values instead of consuming all gas. diff --git a/Cargo.lock b/Cargo.lock index dab5f9f..f08a676 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9035,16 +9035,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "revive-explorer" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap", - "num_cpus", - "revive-yul", -] - [[package]] name = "revive-integration" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 55243d2..38f5c65 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,6 @@ revive-build-utils = { version = "0.2.0", path = "crates/build-utils" } revive-builtins = { version = "0.1.0", path = "crates/builtins" } revive-common = { version = "0.3.0", path = "crates/common" } revive-differential = { version = "0.3.0", path = "crates/differential" } -revive-explorer = { version = "0.1.0", path = "crates/explore" } revive-integration = { version = "0.4.0", path = "crates/integration" } revive-linker = { version = "0.3.0", path = "crates/linker" } revive-llvm-context = { version = "0.6.0", path = "crates/llvm-context" } diff --git a/Makefile b/Makefile index fbf5f8b..4bd831e 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,6 @@ install-llvm-builder \ install-llvm \ install-revive-runner \ - install-revive-explorer \ format \ clippy \ doc \ @@ -49,9 +48,6 @@ install-llvm: install-llvm-builder install-revive-runner: cargo install --locked --force --path crates/runner --no-default-features -install-revive-explorer: - cargo install --locked --force --path crates/explorer --no-default-features - format: cargo fmt --all --check @@ -68,7 +64,7 @@ machete: cargo install cargo-machete cargo machete -test: format clippy machete test-workspace install-revive-runner install-revive-explorer doc test-book +test: format clippy machete test-workspace install-revive-runner doc test-book test-integration: install-bin cargo test --package revive-integration diff --git a/crates/explorer/Cargo.toml b/crates/explorer/Cargo.toml deleted file mode 100644 index 3ba9075..0000000 --- a/crates/explorer/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "revive-explorer" -version = "0.1.0" -license.workspace = true -edition.workspace = true -repository.workspace = true -authors.workspace = true -description = "Helper utility to inspect debug builds" - -[[bin]] -name = "revive-explorer" -path = "src/main.rs" - -[dependencies] -anyhow = { workspace = true } -clap = { workspace = true, features = ["help", "std", "derive"] } -num_cpus = { workspace = true } - -revive-yul = { workspace = true } - diff --git a/crates/explorer/README.md b/crates/explorer/README.md deleted file mode 100644 index 9cb7d37..0000000 --- a/crates/explorer/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# revive-explorer - -The `revive-explorer` is a helper utility for exploring the compilers YUL lowering unit. - -It analyzes a given shared objects from the debug dump and outputs: -- The count of each YUL statement translated. -- A per YUL statement break-down of bytecode size contributed per. -- Estimated `yul-phaser` cost parameters. - -Example: - -``` -statements count: - block 532 - Caller 20 - Not 73 - Gas 24 - Shr 2 - ... - Shl 259 - SetImmutable 2 - CodeSize 1 - CallDataLoad 87 - Return 56 -bytes per statement: - Or 756 - CodeCopy 158 - Log3 620 - Return 1562 - MStore 36128 - ... - ReturnDataCopy 2854 - DataOffset 28 - assignment 1194 - Number 540 - CallValue 4258 -yul-phaser parameters: - --break-cost 1 - --variable-declaration-cost 3 - --function-call-cost 8 - --if-cost 4 - --expression-statement-cost 6 - --function-definition-cost 11 - --switch-cost 3 - --block-cost 1 - --leave-cost 1 - --assignment-cost 1 -``` - diff --git a/crates/explorer/src/dwarfdump.rs b/crates/explorer/src/dwarfdump.rs deleted file mode 100644 index 77c0518..0000000 --- a/crates/explorer/src/dwarfdump.rs +++ /dev/null @@ -1,68 +0,0 @@ -//! The `llvm-dwarfdump` utility helper library. - -use std::{ - path::{Path, PathBuf}, - process::{Command, Stdio}, -}; - -pub static EXECUTABLE: &str = "llvm-dwarfdump"; -pub static DEBUG_LINES_ARGUMENTS: [&str; 1] = ["--debug-line"]; -pub static SOURCE_FILE_ARGUMENTS: [&str; 1] = ["--show-sources"]; - -/// Calls the `llvm-dwarfdump` tool to extract debug line information -/// from the shared object at `path`. Returns the output. -/// -/// Provide `Some(dwarfdump_exectuable)` to override the default executable. -pub fn debug_lines( - shared_object: &Path, - dwarfdump_executable: &Option, -) -> anyhow::Result { - dwarfdump(shared_object, dwarfdump_executable, &DEBUG_LINES_ARGUMENTS) -} - -/// Calls the `llvm-dwarfdump` tool to extract the source file name. -/// Returns the source file path. -/// -/// Provide `Some(dwarfdump_exectuable)` to override the default executable. -pub fn source_file( - shared_object: &Path, - dwarfdump_executable: &Option, -) -> anyhow::Result { - let output = dwarfdump(shared_object, dwarfdump_executable, &SOURCE_FILE_ARGUMENTS)?; - let output = output.trim(); - - if output.is_empty() { - anyhow::bail!( - "the shared object at path `{}` doesn't contain the source file name. Hint: compile with debug information (-g)?", - shared_object.display() - ); - } - - Ok(output.into()) -} - -/// The internal `llvm-dwarfdump` helper function. -fn dwarfdump( - shared_object: &Path, - dwarfdump_executable: &Option, - arguments: &[&str], -) -> anyhow::Result { - let executable = dwarfdump_executable - .to_owned() - .unwrap_or_else(|| PathBuf::from(EXECUTABLE)); - - let output = Command::new(executable) - .args(arguments) - .arg(shared_object) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - .wait_with_output()?; - - if !output.status.success() { - anyhow::bail!(String::from_utf8_lossy(&output.stderr).to_string()); - } - - Ok(String::from_utf8_lossy(&output.stdout).to_string()) -} diff --git a/crates/explorer/src/dwarfdump_analyzer.rs b/crates/explorer/src/dwarfdump_analyzer.rs deleted file mode 100644 index 00dadb3..0000000 --- a/crates/explorer/src/dwarfdump_analyzer.rs +++ /dev/null @@ -1,222 +0,0 @@ -//! The core dwarf dump analyzer library. - -use std::{ - collections::HashMap, - path::{Path, PathBuf}, -}; - -use revive_yul::lexer::token::location::Location; - -use crate::location_mapper::{self, LocationMapper}; - -/// The dwarf dump analyzer. -/// -/// Loads debug information from `llvm-dwarfdump` and calculates statistics -/// about the compiled YUL statements: -/// - Statements count -/// - Per-statement -#[derive(Debug, Default)] -pub struct DwarfdumpAnalyzer { - /// The YUL source file path. - source: PathBuf, - - /// The YUL location to statements map. - location_map: HashMap, - - /// The `llvm-dwarfdump --debug-lines` output. - debug_lines: String, - - /// The observed statements. - statements_count: HashMap, - /// The observed statement to instructions size. - statements_size: HashMap, -} - -impl DwarfdumpAnalyzer { - /// The debug info analyzer constructor. - /// - /// `source` is the path to the YUL source file. - /// `debug_lines` is the `llvm-dwarfdump --debug-lines` output. - pub fn new(source: &Path, debug_lines: String) -> Self { - Self { - source: source.to_path_buf(), - debug_lines, - ..Default::default() - } - } - - /// Run the analysis. - pub fn analyze(&mut self) -> anyhow::Result<()> { - self.map_locations()?; - self.analyze_statements()?; - Ok(()) - } - - /// Populate the maps so that we can always unwrap later. - fn map_locations(&mut self) -> anyhow::Result<()> { - self.location_map = LocationMapper::map_locations(&self.source)?; - - self.statements_count = HashMap::with_capacity(self.location_map.len()); - self.statements_size = HashMap::with_capacity(self.location_map.len()); - - for statement in self.location_map.values() { - if !self.statements_size.contains_key(statement) { - self.statements_size.insert(statement.clone(), 0); - } - - *self.statements_count.entry(statement.clone()).or_insert(0) += 1; - } - - Ok(()) - } - - /// Analyze how much bytes of insturctions each statement contributes. - fn analyze_statements(&mut self) -> anyhow::Result<()> { - let mut previous_offset = 0; - let mut previous_location = Location::new(0, 0); - - for line in self - .debug_lines - .lines() - .skip_while(|line| !line.starts_with("Address")) - .skip(2) - { - let mut parts = line.split_whitespace(); - let (Some(offset), Some(line), Some(column)) = - (parts.next(), parts.next(), parts.next()) - else { - continue; - }; - - let current_offset = u64::from_str_radix(offset.trim_start_matches("0x"), 16)?; - let mut current_location = Location::new(line.parse()?, column.parse()?); - - // TODO: A bug? Needs further investigation. - if current_location.line == 0 && current_location.column != 0 { - current_location.line = previous_location.line; - } - - if let Some(statement) = self.location_map.get(&previous_location) { - let contribution = current_offset - previous_offset; - *self.statements_size.get_mut(statement).unwrap() += contribution; - } - - previous_offset = current_offset; - previous_location = current_location; - } - - Ok(()) - } - - /// Print the per-statement count break-down. - pub fn display_statement_count(&self) { - println!("statements count:"); - for (statement, count) in self.statements_count.iter() { - println!("\t{statement} {count}"); - } - } - - /// Print the per-statement byte size contribution break-down. - pub fn display_statement_size(&self) { - println!("bytes per statement:"); - for (statement, size) in self.statements_size.iter() { - println!("\t{statement} {size}"); - } - } - - /// Print the estimated `yul-phaser` cost parameters. - pub fn display_phaser_costs(&self, yul_phaser_scale: u64) { - println!("yul-phaser parameters:"); - for (parameter, cost) in self.phaser_costs(yul_phaser_scale) { - println!("\t{parameter} {cost}"); - } - } - - /// Estimate the `yul-phaser` costs using the simplified weight function: - /// `Total size / toal count = cost` - pub fn phaser_costs(&self, yul_phaser_scale: u64) -> Vec<(String, u64)> { - let mut costs: HashMap = HashMap::with_capacity(16); - for (statement, count) in self - .statements_count - .iter() - .filter(|(_, count)| **count > 0) - { - let size = self.statements_size.get(statement).unwrap(); - let cost = match statement.as_str() { - location_mapper::FOR => "--for-loop-cost", - location_mapper::OTHER => continue, - location_mapper::INTERNAL => continue, - location_mapper::BLOCK => "--block-cost", - location_mapper::FUNCTION_CALL => "--function-call-cost", - location_mapper::IF => "--if-cost", - location_mapper::SWITCH => "--switch-cost", - location_mapper::DECLARATION => "--variable-declaration-cost", - location_mapper::ASSIGNMENT => "--assignment-cost", - location_mapper::FUNCTION_DEFINITION => "--function-definition-cost", - location_mapper::IDENTIFIER => "--identifier-cost", - location_mapper::LITERAL => "--literal-cost", - _ => "--expression-statement-cost", - }; - - let entry = costs.entry(cost.to_string()).or_default(); - entry.0 += count; - entry.1 += size; - } - - let costs = costs - .iter() - .map(|(cost, (count, size))| { - let ratio = *size / *count as u64; - (cost.to_string(), ratio.min(100)) - }) - .collect::>(); - - let scaled_costs = scale_to( - costs - .iter() - .map(|(_, ratio)| *ratio) - .collect::>() - .as_slice(), - yul_phaser_scale, - ); - - costs - .iter() - .zip(scaled_costs) - .map(|((cost, _), scaled_ratio)| (cost.to_string(), scaled_ratio)) - .collect() - } -} - -/// Given a slice of u64 values, returns a `Vec` where each element -/// is linearly scaled into the closed interval [1, 10]. -fn scale_to(data: &[u64], scale_max: u64) -> Vec { - if data.is_empty() { - return Vec::new(); - } - - let mut min = data[0]; - let mut max = data[0]; - for &x in &data[1..] { - if x < min { - min = x; - } - if x > max { - max = x; - } - } - if max < scale_max { - return data.to_vec(); - } - - let range = max - min; - data.iter() - .map(|&x| { - if range == 0 { - 1 - } else { - 1 + (x - min) * scale_max / range - } - }) - .collect() -} diff --git a/crates/explorer/src/lib.rs b/crates/explorer/src/lib.rs deleted file mode 100644 index bdb725d..0000000 --- a/crates/explorer/src/lib.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! The revive explorer leverages debug info to get insights into emitted code. - -pub mod dwarfdump; -pub mod dwarfdump_analyzer; -pub mod location_mapper; -pub mod yul_phaser; diff --git a/crates/explorer/src/location_mapper.rs b/crates/explorer/src/location_mapper.rs deleted file mode 100644 index f170cf0..0000000 --- a/crates/explorer/src/location_mapper.rs +++ /dev/null @@ -1,123 +0,0 @@ -//! The location mapper utility maps YUL source locations to AST statements. - -use std::{collections::HashMap, path::Path}; - -use revive_yul::{ - lexer::{token::location::Location, Lexer}, - parser::{ - identifier::Identifier, - statement::{ - assignment::Assignment, - block::Block, - expression::{function_call::FunctionCall, literal::Literal}, - for_loop::ForLoop, - function_definition::FunctionDefinition, - if_conditional::IfConditional, - object::Object, - switch::Switch, - variable_declaration::VariableDeclaration, - }, - }, - visitor::{AstNode, AstVisitor}, -}; - -/// Code attributed to an unknown location. -pub const OTHER: &str = "other"; -/// Code attributed to a compiler internal location. -pub const INTERNAL: &str = "internal"; -/// Code attributed to a block. -pub const BLOCK: &str = "block"; -/// Code attributed to a function call. -pub const FUNCTION_CALL: &str = "function_call"; -/// Code attributed to a for loop. -pub const FOR: &str = "for"; -/// Code attributed to an if statement. -pub const IF: &str = "if"; -/// Code attributed to a switch statement. -pub const SWITCH: &str = "switch"; -/// Code attributed to a variable declaration. -pub const DECLARATION: &str = "let"; -/// Code attributed to a variable assignement. -pub const ASSIGNMENT: &str = "assignment"; -/// Code attributed to a function definition. -pub const FUNCTION_DEFINITION: &str = "function_definition"; -/// Code attributed to an identifier. -pub const IDENTIFIER: &str = "identifier"; -/// Code attributed to a literal. -pub const LITERAL: &str = "literal"; - -/// The location to statements mapper. -pub struct LocationMapper(HashMap); - -impl LocationMapper { - /// Construct a node location map from the given YUL `source` file. - pub fn map_locations(source: &Path) -> anyhow::Result> { - let mut lexer = Lexer::new(std::fs::read_to_string(source)?); - let ast = Object::parse(&mut lexer, None).map_err(|error| { - anyhow::anyhow!("Contract `{}` parsing error: {:?}", source.display(), error) - })?; - - let mut location_map = Self(Default::default()); - ast.accept(&mut location_map); - location_map.0.insert(Location::new(0, 0), OTHER.into()); - location_map.0.insert(Location::new(1, 0), INTERNAL.into()); - - Ok(location_map.0) - } -} - -impl AstVisitor for LocationMapper { - fn visit(&mut self, node: &impl AstNode) { - node.visit_children(self); - } - - fn visit_block(&mut self, node: &Block) { - node.visit_children(self); - self.0.insert(node.location, BLOCK.into()); - } - - fn visit_assignment(&mut self, node: &Assignment) { - node.visit_children(self); - self.0.insert(node.location, ASSIGNMENT.into()); - } - - fn visit_if_conditional(&mut self, node: &IfConditional) { - node.visit_children(self); - self.0.insert(node.location, IF.into()); - } - - fn visit_variable_declaration(&mut self, node: &VariableDeclaration) { - node.visit_children(self); - self.0.insert(node.location, DECLARATION.into()); - } - - fn visit_function_call(&mut self, node: &FunctionCall) { - node.visit_children(self); - self.0.insert(node.location, node.name.to_string()); - } - - fn visit_function_definition(&mut self, node: &FunctionDefinition) { - node.visit_children(self); - self.0.insert(node.location, FUNCTION_DEFINITION.into()); - } - - fn visit_identifier(&mut self, node: &Identifier) { - node.visit_children(self); - self.0.insert(node.location, IDENTIFIER.into()); - } - - fn visit_literal(&mut self, node: &Literal) { - node.visit_children(self); - self.0.insert(node.location, LITERAL.into()); - } - - fn visit_for_loop(&mut self, node: &ForLoop) { - node.visit_children(self); - self.0.insert(node.location, FOR.into()); - } - - fn visit_switch(&mut self, node: &Switch) { - node.visit_children(self); - self.0.insert(node.location, SWITCH.into()); - } -} diff --git a/crates/explorer/src/main.rs b/crates/explorer/src/main.rs deleted file mode 100644 index 10036d1..0000000 --- a/crates/explorer/src/main.rs +++ /dev/null @@ -1,59 +0,0 @@ -use std::path::PathBuf; - -use clap::Parser; - -use revive_explorer::{dwarfdump, dwarfdump_analyzer::DwarfdumpAnalyzer, yul_phaser}; - -/// The `revive-explorer` is a helper utility for exploring the compilers YUL lowering unit. -/// -/// It analyzes a given shared objects from the debug dump and outputs: -/// - The count of each YUL statement translated. -/// - A per YUL statement break-down of bytecode size contributed per. -/// - Estimated `yul-phaser` cost parameters. -/// -/// Note: This tool might not be fully accurate, especially when the code was optimized. -#[derive(Parser, Debug)] -#[command(version, about, long_about = None)] -struct Args { - /// Path of the dwarfdump executable. - #[arg(short, long)] - dwarfdump: Option, - - /// The YUL phaser cost scale maximum value. - #[arg(short, long, default_value_t = 10)] - cost_scale: u64, - - /// Run the provided yul-phaser executable using the estimated costs. - #[arg(short, long)] - yul_phaser: Option, - - /// Path of the shared object to analyze. - /// It must have been compiled with debug info (-g). - file: PathBuf, -} - -fn main() -> anyhow::Result<()> { - let args = Args::parse(); - - let source_file = dwarfdump::source_file(&args.file, &args.dwarfdump)?; - let debug_lines = dwarfdump::debug_lines(&args.file, &args.dwarfdump)?; - let mut analyzer = DwarfdumpAnalyzer::new(source_file.as_path(), debug_lines); - - analyzer.analyze()?; - - if let Some(path) = args.yul_phaser.as_ref() { - yul_phaser::run( - path, - source_file.as_path(), - analyzer.phaser_costs(args.cost_scale).as_slice(), - num_cpus::get() / 2, // TODO: should be configurable. - )?; - return Ok(()); - } - - analyzer.display_statement_count(); - analyzer.display_statement_size(); - analyzer.display_phaser_costs(args.cost_scale); - - Ok(()) -} diff --git a/crates/explorer/src/yul_phaser.rs b/crates/explorer/src/yul_phaser.rs deleted file mode 100644 index f994c52..0000000 --- a/crates/explorer/src/yul_phaser.rs +++ /dev/null @@ -1,79 +0,0 @@ -//! The revive explorer YUL phaser utility library. -//! -//! This can be used to invoke the `yul-phaser` utility, -//! used to find better YUL optimizer sequences. - -use std::{ - path::{Path, PathBuf}, - process::{Command, Stdio}, - thread, - time::{SystemTime, UNIX_EPOCH}, -}; - -/// The `yul-phaser` sane default arguments: -/// - Less verbose output. -/// - Sufficient rounds. -/// - Sufficient random population start. -const ARGUMENTS: [&str; 6] = [ - "--hide-round", - "--rounds", - "1000", - "--random-population", - "100", - "--show-only-top-chromosome", -]; - -/// Run multiple YUL phaser executables in parallel. -pub fn run( - executable: &Path, - source: &Path, - costs: &[(String, u64)], - n_threads: usize, -) -> anyhow::Result<()> { - let mut handles = Vec::with_capacity(n_threads); - - for n in 0..n_threads { - let executable = executable.to_path_buf(); - let source = source.to_path_buf(); - let costs = costs.to_vec(); - - handles.push(thread::spawn(move || { - spawn_process(executable, source, costs, n) - })); - } - - for handle in handles { - let _ = handle.join(); - } - - Ok(()) -} - -/// The `yul-phaser` process spawning helper function. -fn spawn_process( - executable: PathBuf, - source: PathBuf, - costs: Vec<(String, u64)>, - seed: usize, -) -> anyhow::Result<()> { - let cost_parameters = costs - .iter() - .flat_map(|(parameter, cost)| vec![parameter.clone(), cost.to_string()]); - - let secs = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Time went backwards") - .as_secs(); - - Command::new(executable) - .args(cost_parameters) - .args(ARGUMENTS) - .arg("--seed") - .arg((seed + secs as usize).to_string()) - .arg(source) - .stdin(Stdio::null()) - .spawn()? - .wait()?; - - Ok(()) -}