diff --git a/Cargo.lock b/Cargo.lock index 77f248d..954a832 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2936,6 +2936,22 @@ dependencies = [ "sp-crypto-hashing", ] +[[package]] +name = "frame-decode" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c470df86cf28818dd3cd2fc4667b80dbefe2236c722c3dc1d09e7c6c82d6dfcd" +dependencies = [ + "frame-metadata", + "parity-scale-codec", + "scale-decode", + "scale-encode", + "scale-info", + "scale-type-resolver", + "sp-crypto-hashing", + "thiserror 2.0.12", +] + [[package]] name = "frame-metadata" version = "23.0.0" @@ -5681,6 +5697,7 @@ dependencies = [ "serde_yaml_ng", "sp-core", "sp-runtime", + "subxt 0.44.0", "temp-dir", "tokio", "tower 0.5.2", @@ -7350,11 +7367,48 @@ dependencies = [ "serde", "serde_json", "sp-crypto-hashing", - "subxt-core", - "subxt-lightclient", - "subxt-macro", - "subxt-metadata", - "subxt-rpcs", + "subxt-core 0.43.0", + "subxt-lightclient 0.43.0", + "subxt-macro 0.43.0", + "subxt-metadata 0.43.0", + "subxt-rpcs 0.43.0", + "thiserror 2.0.12", + "tokio", + "tokio-util", + "tracing", + "url", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "subxt" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddbf938ac1d86a361a84709a71cdbae5d87f370770b563651d1ec052eed9d0b4" +dependencies = [ + "async-trait", + "derive-where", + "either", + "frame-metadata", + "futures", + "hex", + "jsonrpsee", + "parity-scale-codec", + "primitive-types 0.13.1", + "scale-bits", + "scale-decode", + "scale-encode", + "scale-info", + "scale-value", + "serde", + "serde_json", + "sp-crypto-hashing", + "subxt-core 0.44.0", + "subxt-lightclient 0.44.0", + "subxt-macro 0.44.0", + "subxt-metadata 0.44.0", + "subxt-rpcs 0.44.0", "thiserror 2.0.12", "tokio", "tokio-util", @@ -7376,7 +7430,24 @@ dependencies = [ "quote", "scale-info", "scale-typegen", - "subxt-metadata", + "subxt-metadata 0.43.0", + "syn 2.0.101", + "thiserror 2.0.12", +] + +[[package]] +name = "subxt-codegen" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c250ad8cd102d40ae47977b03295a2ff791375f30ddc7474d399fb56efb793b" +dependencies = [ + "heck", + "parity-scale-codec", + "proc-macro2", + "quote", + "scale-info", + "scale-typegen", + "subxt-metadata 0.44.0", "syn 2.0.101", "thiserror 2.0.12", ] @@ -7390,7 +7461,7 @@ dependencies = [ "base58", "blake2", "derive-where", - "frame-decode", + "frame-decode 0.8.3", "frame-metadata", "hashbrown 0.14.5", "hex", @@ -7406,7 +7477,37 @@ dependencies = [ "serde", "serde_json", "sp-crypto-hashing", - "subxt-metadata", + "subxt-metadata 0.43.0", + "thiserror 2.0.12", + "tracing", +] + +[[package]] +name = "subxt-core" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5705c5b420294524e41349bf23c6b11aa474ce731de7317f4153390e1927f702" +dependencies = [ + "base58", + "blake2", + "derive-where", + "frame-decode 0.9.0", + "frame-metadata", + "hashbrown 0.14.5", + "hex", + "impl-serde", + "keccak-hash", + "parity-scale-codec", + "primitive-types 0.13.1", + "scale-bits", + "scale-decode", + "scale-encode", + "scale-info", + "scale-value", + "serde", + "serde_json", + "sp-crypto-hashing", + "subxt-metadata 0.44.0", "thiserror 2.0.12", "tracing", ] @@ -7428,6 +7529,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "subxt-lightclient" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e02732a6c9ae46bc282c1a741b3d3e494021b3e87e7e92cfb3620116d92911" +dependencies = [ + "futures", + "futures-util", + "serde", + "serde_json", + "smoldot-light", + "thiserror 2.0.12", + "tokio", + "tokio-stream", + "tracing", +] + [[package]] name = "subxt-macro" version = "0.43.0" @@ -7439,9 +7557,26 @@ dependencies = [ "proc-macro-error2", "quote", "scale-typegen", - "subxt-codegen", - "subxt-metadata", - "subxt-utils-fetchmetadata", + "subxt-codegen 0.43.0", + "subxt-metadata 0.43.0", + "subxt-utils-fetchmetadata 0.43.0", + "syn 2.0.101", +] + +[[package]] +name = "subxt-macro" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501bf358698f5ab02a6199a1fcd3f1b482e2f5b6eb5d185411e6a74a175ec8e8" +dependencies = [ + "darling 0.20.11", + "parity-scale-codec", + "proc-macro-error2", + "quote", + "scale-typegen", + "subxt-codegen 0.44.0", + "subxt-metadata 0.44.0", + "subxt-utils-fetchmetadata 0.44.0", "syn 2.0.101", ] @@ -7451,7 +7586,22 @@ version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c134068711c0c46906abc0e6e4911204420331530738e18ca903a5469364d9f" dependencies = [ - "frame-decode", + "frame-decode 0.8.3", + "frame-metadata", + "hashbrown 0.14.5", + "parity-scale-codec", + "scale-info", + "sp-crypto-hashing", + "thiserror 2.0.12", +] + +[[package]] +name = "subxt-metadata" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01fb7c0bfafad78dda7084c6a2444444744af3bbf7b2502399198b9b4c20eddf" +dependencies = [ + "frame-decode 0.9.0", "frame-metadata", "hashbrown 0.14.5", "parity-scale-codec", @@ -7476,8 +7626,32 @@ dependencies = [ "primitive-types 0.13.1", "serde", "serde_json", - "subxt-core", - "subxt-lightclient", + "subxt-core 0.43.0", + "subxt-lightclient 0.43.0", + "thiserror 2.0.12", + "tokio-util", + "tracing", + "url", +] + +[[package]] +name = "subxt-rpcs" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab68a9c20ecedb0cb7d62d64f884e6add91bb70485783bf40aa8eac5c389c6e0" +dependencies = [ + "derive-where", + "frame-metadata", + "futures", + "hex", + "impl-serde", + "jsonrpsee", + "parity-scale-codec", + "primitive-types 0.13.1", + "serde", + "serde_json", + "subxt-core 0.44.0", + "subxt-lightclient 0.44.0", "thiserror 2.0.12", "tokio-util", "tracing", @@ -7507,7 +7681,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "sp-crypto-hashing", - "subxt-core", + "subxt-core 0.43.0", "thiserror 2.0.12", "zeroize", ] @@ -7523,6 +7697,17 @@ dependencies = [ "thiserror 2.0.12", ] +[[package]] +name = "subxt-utils-fetchmetadata" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e450f6812a653c5a3e63a079aa3b60a3f4c362722753c3222286eaa1800f9002" +dependencies = [ + "hex", + "parity-scale-codec", + "thiserror 2.0.12", +] + [[package]] name = "syn" version = "1.0.109" @@ -9316,7 +9501,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "sp-core", - "subxt", + "subxt 0.43.0", "subxt-signer", "thiserror 1.0.69", "tokio", @@ -9376,7 +9561,7 @@ dependencies = [ "async-trait", "futures", "lazy_static", - "subxt", + "subxt 0.43.0", "subxt-signer", "tokio", "zombienet-configuration", diff --git a/Cargo.toml b/Cargo.toml index d1c48b5..f3a550b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ sha2 = { version = "0.10.9" } sp-core = "36.1.0" sp-runtime = "41.1.0" strum = { version = "0.27.2", features = ["derive"] } +subxt = { version = "0.44.0" } temp-dir = { version = "0.1.16" } tempfile = "3.3" thiserror = "2" diff --git a/assets/revive_metadata.scale b/assets/revive_metadata.scale new file mode 100644 index 0000000..68c0a1e Binary files /dev/null and b/assets/revive_metadata.scale differ diff --git a/crates/core/src/differential_benchmarks/watcher.rs b/crates/core/src/differential_benchmarks/watcher.rs index 96f0098..473f134 100644 --- a/crates/core/src/differential_benchmarks/watcher.rs +++ b/crates/core/src/differential_benchmarks/watcher.rs @@ -106,7 +106,9 @@ impl Watcher { // region:TEMPORARY eprintln!("Watcher information for {}", self.platform_identifier); - eprintln!("block_number,block_timestamp,mined_gas,block_gas_limit,tx_count"); + eprintln!( + "block_number,block_timestamp,mined_gas,block_gas_limit,tx_count,ref_time,max_ref_time,proof_size,max_proof_size" + ); // endregion:TEMPORARY while let Some(block) = blocks_information_stream.next().await { // If the block number is equal to or less than the last block before the @@ -141,12 +143,16 @@ impl Watcher { // reporting in place and then it can be removed. This serves as as way of doing // some very simple reporting for the time being. eprintln!( - "\"{}\",\"{}\",\"{}\",\"{}\",\"{}\"", + "\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\"", block.block_number, block.block_timestamp, block.mined_gas, block.block_gas_limit, - block.transaction_hashes.len() + block.transaction_hashes.len(), + block.ref_time, + block.max_ref_time, + block.proof_size, + block.max_proof_size, ); // endregion:TEMPORARY diff --git a/crates/node-interaction/src/lib.rs b/crates/node-interaction/src/lib.rs index 29907a0..54796a6 100644 --- a/crates/node-interaction/src/lib.rs +++ b/crates/node-interaction/src/lib.rs @@ -97,4 +97,16 @@ pub struct MinedBlockInformation { /// The hashes of the transactions that were mined as part of the block. pub transaction_hashes: Vec, + + /// The ref time for substrate based chains. + pub ref_time: u128, + + /// The max ref time for substrate based chains. + pub max_ref_time: u64, + + /// The proof size for substrate based chains. + pub proof_size: u128, + + /// The max proof size for substrate based chains. + pub max_proof_size: u64, } diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml index dfe0827..c709bf4 100644 --- a/crates/node/Cargo.toml +++ b/crates/node/Cargo.toml @@ -30,6 +30,7 @@ serde_yaml_ng = { workspace = true } sp-core = { workspace = true } sp-runtime = { workspace = true } +subxt = { workspace = true } zombienet-sdk = { workspace = true } [dev-dependencies] diff --git a/crates/node/src/node_implementations/geth.rs b/crates/node/src/node_implementations/geth.rs index 5997fd7..097265d 100644 --- a/crates/node/src/node_implementations/geth.rs +++ b/crates/node/src/node_implementations/geth.rs @@ -535,6 +535,10 @@ impl EthereumNode for GethNode { .as_hashes() .expect("Must be hashes") .to_vec(), + ref_time: 0, + max_ref_time: 0, + proof_size: 0, + max_proof_size: 0, }) }); diff --git a/crates/node/src/node_implementations/lighthouse_geth.rs b/crates/node/src/node_implementations/lighthouse_geth.rs index 5b00145..beca317 100644 --- a/crates/node/src/node_implementations/lighthouse_geth.rs +++ b/crates/node/src/node_implementations/lighthouse_geth.rs @@ -222,6 +222,7 @@ impl LighthouseGethNode { "--ws.port=8546".to_string(), "--ws.api=eth,net,web3,txpool,engine".to_string(), "--ws.origins=*".to_string(), + "--miner.gaslimit=30000000".to_string(), ], consensus_layer_extra_parameters: vec![ "--disable-quic".to_string(), @@ -247,6 +248,8 @@ impl LighthouseGethNode { .collect::>(); serde_json::to_string(&map).unwrap() }, + gas_limit: 30_000_000, + genesis_gaslimit: 30_000_000, }, wait_for_finalization: false, port_publisher: Some(PortPublisherParameters { @@ -754,6 +757,10 @@ impl EthereumNode for LighthouseGethNode { .as_hashes() .expect("Must be hashes") .to_vec(), + ref_time: 0, + max_ref_time: 0, + proof_size: 0, + max_proof_size: 0, }) }); @@ -1045,6 +1052,8 @@ struct NetworkParameters { pub num_validator_keys_per_node: u64, pub genesis_delay: u64, + pub genesis_gaslimit: u64, + pub gas_limit: u64, pub prefunded_accounts: String, } diff --git a/crates/node/src/node_implementations/substrate.rs b/crates/node/src/node_implementations/substrate.rs index 548730d..e4fdec4 100644 --- a/crates/node/src/node_implementations/substrate.rs +++ b/crates/node/src/node_implementations/substrate.rs @@ -28,8 +28,7 @@ use alloy::{ }, }; use anyhow::Context as _; -use async_stream::stream; -use futures::{FutureExt, Stream}; +use futures::{FutureExt, Stream, StreamExt}; use revive_common::EVMVersion; use revive_dt_common::fs::clear_directory; use revive_dt_format::traits::ResolverApi; @@ -39,6 +38,7 @@ use sp_runtime::AccountId32; use revive_dt_config::*; use revive_dt_node_interaction::{EthereumNode, MinedBlockInformation}; +use subxt::{OnlineClient, SubstrateConfig}; use tokio::sync::OnceCell; use tracing::instrument; @@ -508,46 +508,80 @@ impl EthereumNode for SubstrateNode { + '_, >, > { - fn create_stream( - provider: ConcreteProvider>, - ) -> impl Stream { - stream! { - let mut block_number = provider.get_block_number().await.expect("Failed to get the block number"); - loop { - tokio::time::sleep(Duration::from_secs(1)).await; + #[subxt::subxt(runtime_metadata_path = "../../assets/revive_metadata.scale")] + pub mod revive {} - let Ok(Some(block)) = provider.get_block_by_number(BlockNumberOrTag::Number(block_number)).await - else { - continue; - }; + Box::pin(async move { + let substrate_rpc_port = Self::BASE_SUBSTRATE_RPC_PORT + self.id as u16; + let substrate_rpc_url = format!("ws://127.0.0.1:{substrate_rpc_port}"); + let api = OnlineClient::::from_url(substrate_rpc_url) + .await + .context("Failed to create subxt rpc client")?; + let provider = self.provider().await.context("Failed to create provider")?; - block_number += 1; - yield MinedBlockInformation { - block_number: block.number(), - block_timestamp: block.header.timestamp, - mined_gas: block.header.gas_used as _, - block_gas_limit: block.header.gas_limit as _, - transaction_hashes: block + let block_stream = api + .blocks() + .subscribe_all() + .await + .context("Failed to subscribe to blocks")?; + + let mined_block_information_stream = block_stream.filter_map(move |block| { + let api = api.clone(); + let provider = provider.clone(); + + async move { + let substrate_block = block.ok()?; + let revive_block = provider + .get_block_by_number( + BlockNumberOrTag::Number(substrate_block.number() as _), + ) + .await + .expect("TODO: Remove") + .expect("TODO: Remove"); + + let used = api + .storage() + .at(substrate_block.reference()) + .fetch_or_default(&revive::storage().system().block_weight()) + .await + .expect("TODO: Remove"); + + let block_ref_time = (used.normal.ref_time as u128) + + (used.operational.ref_time as u128) + + (used.mandatory.ref_time as u128); + let block_proof_size = (used.normal.proof_size as u128) + + (used.operational.proof_size as u128) + + (used.mandatory.proof_size as u128); + + let limits = api + .constants() + .at(&revive::constants().system().block_weights()) + .expect("TODO: Remove"); + + let max_ref_time = limits.max_block.ref_time; + let max_proof_size = limits.max_block.proof_size; + + Some(MinedBlockInformation { + block_number: substrate_block.number() as _, + block_timestamp: revive_block.header.timestamp, + mined_gas: revive_block.header.gas_used as _, + block_gas_limit: revive_block.header.gas_limit as _, + transaction_hashes: revive_block .transactions .into_hashes() .as_hashes() .expect("Must be hashes") .to_vec(), - }; - }; - } - } + ref_time: block_ref_time, + max_ref_time, + proof_size: block_proof_size, + max_proof_size, + }) + } + }); - Box::pin(async move { - let provider = self - .provider() - .await - .context("Failed to create the provider for a block subscription")?; - - let stream = Box::pin(create_stream(provider)) - as Pin>>; - - Ok(stream) + Ok(Box::pin(mined_block_information_stream) + as Pin>>) }) } diff --git a/crates/node/src/node_implementations/zombienet.rs b/crates/node/src/node_implementations/zombienet.rs index 64fcbd1..8387685 100644 --- a/crates/node/src/node_implementations/zombienet.rs +++ b/crates/node/src/node_implementations/zombienet.rs @@ -55,8 +55,7 @@ use alloy::{ }; use anyhow::Context as _; -use async_stream::stream; -use futures::{FutureExt, Stream}; +use futures::{FutureExt, Stream, StreamExt}; use revive_common::EVMVersion; use revive_dt_common::fs::clear_directory; use revive_dt_config::*; @@ -65,6 +64,7 @@ use revive_dt_node_interaction::{EthereumNode, MinedBlockInformation}; use serde_json::{Value as JsonValue, json}; use sp_core::crypto::Ss58Codec; use sp_runtime::AccountId32; +use subxt::{OnlineClient, SubstrateConfig}; use tokio::sync::OnceCell; use tracing::instrument; use zombienet_sdk::{LocalFileSystem, NetworkConfigBuilder, NetworkConfigExt}; @@ -564,46 +564,79 @@ impl EthereumNode for ZombienetNode { + '_, >, > { - fn create_stream( - provider: ConcreteProvider>, - ) -> impl Stream { - stream! { - let mut block_number = provider.get_block_number().await.expect("Failed to get the block number"); - loop { - tokio::time::sleep(Duration::from_secs(1)).await; + #[subxt::subxt(runtime_metadata_path = "../../assets/revive_metadata.scale")] + pub mod revive {} - let Ok(Some(block)) = provider.get_block_by_number(BlockNumberOrTag::Number(block_number)).await - else { - continue; - }; + Box::pin(async move { + let substrate_rpc_url = format!("ws://127.0.0.1:{}", self.node_rpc_port.unwrap()); + let api = OnlineClient::::from_url(substrate_rpc_url) + .await + .context("Failed to create subxt rpc client")?; + let provider = self.provider().await.context("Failed to create provider")?; - block_number += 1; - yield MinedBlockInformation { - block_number: block.number(), - block_timestamp: block.header.timestamp, - mined_gas: block.header.gas_used as _, - block_gas_limit: block.header.gas_limit as _, - transaction_hashes: block + let block_stream = api + .blocks() + .subscribe_all() + .await + .context("Failed to subscribe to blocks")?; + + let mined_block_information_stream = block_stream.filter_map(move |block| { + let api = api.clone(); + let provider = provider.clone(); + + async move { + let substrate_block = block.ok()?; + let revive_block = provider + .get_block_by_number( + BlockNumberOrTag::Number(substrate_block.number() as _), + ) + .await + .expect("TODO: Remove") + .expect("TODO: Remove"); + + let used = api + .storage() + .at(substrate_block.reference()) + .fetch_or_default(&revive::storage().system().block_weight()) + .await + .expect("TODO: Remove"); + + let block_ref_time = (used.normal.ref_time as u128) + + (used.operational.ref_time as u128) + + (used.mandatory.ref_time as u128); + let block_proof_size = (used.normal.proof_size as u128) + + (used.operational.proof_size as u128) + + (used.mandatory.proof_size as u128); + + let limits = api + .constants() + .at(&revive::constants().system().block_weights()) + .expect("TODO: Remove"); + + let max_ref_time = limits.max_block.ref_time; + let max_proof_size = limits.max_block.proof_size; + + Some(MinedBlockInformation { + block_number: substrate_block.number() as _, + block_timestamp: revive_block.header.timestamp, + mined_gas: revive_block.header.gas_used as _, + block_gas_limit: revive_block.header.gas_limit as _, + transaction_hashes: revive_block .transactions .into_hashes() .as_hashes() .expect("Must be hashes") .to_vec(), - }; - }; - } - } + ref_time: block_ref_time, + max_ref_time, + proof_size: block_proof_size, + max_proof_size, + }) + } + }); - Box::pin(async move { - let provider = self - .provider() - .await - .context("Failed to create the provider for a block subscription")?; - - let stream = Box::pin(create_stream(provider)) - as Pin>>; - - Ok(stream) + Ok(Box::pin(mined_block_information_stream) + as Pin>>) }) } diff --git a/crates/node/src/provider_utils/provider.rs b/crates/node/src/provider_utils/provider.rs index f10b3b6..ba7a2c5 100644 --- a/crates/node/src/provider_utils/provider.rs +++ b/crates/node/src/provider_utils/provider.rs @@ -44,7 +44,7 @@ where // requests at any point of time and no more than that. This is done in an effort to stabilize // the framework from some of the interment issues that we've been seeing related to RPC calls. static GLOBAL_CONCURRENCY_LIMITER_LAYER: LazyLock = - LazyLock::new(|| ConcurrencyLimiterLayer::new(500)); + LazyLock::new(|| ConcurrencyLimiterLayer::new(1000)); let client = ClientBuilder::default() .layer(GLOBAL_CONCURRENCY_LIMITER_LAYER.clone())