Port transaction pool benchmark (#738)

Adds a benchmark to measure transaction throughput.
2026-05-30 04:41:03 +00:00 · 2021-11-11 10:30:54 +01:00
parent 3ca9905166
commit 2691e1928e
7 changed files with 432 additions and 75 deletions
@@ -14,9 +14,11 @@ tokio = { version = "1.10", features = ["macros"] }
 # Substrate
 frame-system = { git = "https://github.com/paritytech/substrate", branch = "master" }
 pallet-transaction-payment = { git = "https://github.com/paritytech/substrate", branch = "master" }
+frame-system-rpc-runtime-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-basic-authorship = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-chain-spec = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sc-transaction-pool-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-consensus = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-executor = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" }
@@ -33,6 +35,7 @@ sp-runtime = { git = "https://github.com/paritytech/substrate", default-features
 sp-state-machine = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-timestamp = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-trie = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" }
 substrate-test-client = { git = "https://github.com/paritytech/substrate", branch = "master" }

 # Polkadot
@@ -50,6 +53,8 @@ cumulus-primitives-parachain-inherent = { path = "../../primitives/parachain-inh
 cumulus-test-runtime = { path = "../runtime" }
 cumulus-test-relay-validation-worker-provider = { path = "../relay-validation-worker-provider" }

+criterion = { version = "0.3.5", features = [ "async_tokio" ] }
+
 # RPC related dependencies
 jsonrpc-core = "18.0.0"

@@ -62,3 +67,7 @@ polkadot-test-service = { git = "https://github.com/paritytech/polkadot", branch
 # Substrate dependencies
 sc-cli = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.12" }
 substrate-test-utils = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.12" }
+
+[[bench]]
+name = "transaction_throughput"
+harness = false
@@ -0,0 +1,253 @@
+// This file is part of Cumulus.
+
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput};
+use cumulus_test_runtime::{AccountId, BalancesCall, SudoCall};
+use futures::{future, join, StreamExt};
+use polkadot_service::polkadot_runtime::constants::currency::DOLLARS;
+use sc_transaction_pool_api::{TransactionPool as _, TransactionSource, TransactionStatus};
+use sp_core::{crypto::Pair, sr25519};
+use sp_runtime::{generic::BlockId, OpaqueExtrinsic};
+
+use cumulus_primitives_core::ParaId;
+use cumulus_test_service::{
+	construct_extrinsic, fetch_nonce, initial_head_data, Client, Keyring::*, TransactionPool,
+};
+
+fn create_accounts(num: usize) -> Vec<sr25519::Pair> {
+	(0..num)
+		.map(|i| {
+			Pair::from_string(&format!("{}/{}", Alice.to_seed(), i), None)
+				.expect("Creates account pair")
+		})
+		.collect()
+}
+
+/// Create the extrinsics that will initialize the accounts from the sudo account (Alice).
+///
+/// `start_nonce` is the current nonce of Alice.
+fn create_account_extrinsics(client: &Client, accounts: &[sr25519::Pair]) -> Vec<OpaqueExtrinsic> {
+	let start_nonce = fetch_nonce(client, Alice.public());
+
+	accounts
+		.iter()
+		.enumerate()
+		.map(|(i, a)| {
+			vec![
+				// Reset the nonce by removing any funds
+				construct_extrinsic(
+					client,
+					SudoCall::sudo {
+						call: Box::new(
+							BalancesCall::set_balance {
+								who: AccountId::from(a.public()).into(),
+								new_free: 0,
+								new_reserved: 0,
+							}
+							.into(),
+						),
+					},
+					Alice.pair(),
+					Some(start_nonce + (i as u32) * 2),
+				),
+				// Give back funds
+				construct_extrinsic(
+					client,
+					SudoCall::sudo {
+						call: Box::new(
+							BalancesCall::set_balance {
+								who: AccountId::from(a.public()).into(),
+								new_free: 1_000_000 * DOLLARS,
+								new_reserved: 0,
+							}
+							.into(),
+						),
+					},
+					Alice.pair(),
+					Some(start_nonce + (i as u32) * 2 + 1),
+				),
+			]
+		})
+		.flatten()
+		.map(OpaqueExtrinsic::from)
+		.collect()
+}
+
+fn create_benchmark_extrinsics(
+	client: &Client,
+	accounts: &[sr25519::Pair],
+	extrinsics_per_account: usize,
+) -> Vec<OpaqueExtrinsic> {
+	accounts
+		.iter()
+		.map(|account| {
+			(0..extrinsics_per_account).map(move |nonce| {
+				construct_extrinsic(
+					client,
+					BalancesCall::transfer { dest: Bob.to_account_id().into(), value: 1 * DOLLARS },
+					account.clone(),
+					Some(nonce as u32),
+				)
+			})
+		})
+		.flatten()
+		.map(OpaqueExtrinsic::from)
+		.collect()
+}
+
+async fn submit_tx_and_wait_for_inclusion(
+	tx_pool: &TransactionPool,
+	tx: OpaqueExtrinsic,
+	client: &Client,
+	wait_for_finalized: bool,
+) {
+	let best_hash = client.chain_info().best_hash;
+
+	let mut watch = tx_pool
+		.submit_and_watch(&BlockId::Hash(best_hash), TransactionSource::External, tx.clone())
+		.await
+		.expect("Submits tx to pool")
+		.fuse();
+
+	loop {
+		match watch.select_next_some().await {
+			TransactionStatus::Finalized(_) => break,
+			TransactionStatus::InBlock(_) if !wait_for_finalized => break,
+			_ => {},
+		}
+	}
+}
+
+fn transaction_throughput_benchmarks(c: &mut Criterion) {
+	sp_tracing::try_init_simple();
+	let mut builder = sc_cli::LoggerBuilder::new("");
+	builder.with_colors(false);
+	let _ = builder.init();
+
+	let para_id = ParaId::from(100);
+	let runtime = tokio::runtime::Runtime::new().expect("Creates tokio runtime");
+	let tokio_handle = runtime.handle();
+
+	// Start alice
+	let alice = cumulus_test_service::run_relay_chain_validator_node(
+		tokio_handle.clone(),
+		Alice,
+		|| {},
+		vec![],
+	);
+
+	// Start bob
+	let bob = cumulus_test_service::run_relay_chain_validator_node(
+		tokio_handle.clone(),
+		Bob,
+		|| {},
+		vec![alice.addr.clone()],
+	);
+
+	// Register parachain
+	runtime
+		.block_on(
+			alice.register_parachain(
+				para_id,
+				cumulus_test_service::runtime::WASM_BINARY
+					.expect("You need to build the WASM binary to run this test!")
+					.to_vec(),
+				initial_head_data(para_id),
+			),
+		)
+		.unwrap();
+
+	// Run charlie as parachain collator
+	let charlie = runtime.block_on(
+		cumulus_test_service::TestNodeBuilder::new(para_id, tokio_handle.clone(), Charlie)
+			.enable_collator()
+			.connect_to_relay_chain_nodes(vec![&alice, &bob])
+			.build(),
+	);
+
+	// Run dave as parachain collator
+	let dave = runtime.block_on(
+		cumulus_test_service::TestNodeBuilder::new(para_id, tokio_handle.clone(), Dave)
+			.enable_collator()
+			.connect_to_parachain_node(&charlie)
+			.connect_to_relay_chain_nodes(vec![&alice, &bob])
+			.build(),
+	);
+
+	runtime.block_on(dave.wait_for_blocks(1));
+
+	let mut group = c.benchmark_group("Transaction pool");
+	let account_num = 10;
+	let extrinsics_per_account = 20;
+	group.sample_size(10);
+	group.throughput(Throughput::Elements(account_num as u64 * extrinsics_per_account as u64));
+
+	let accounts = create_accounts(account_num);
+	let mut counter = 1;
+
+	let benchmark_handle = tokio_handle.clone();
+	group.bench_function(
+		format!("{} transfers from {} accounts", account_num * extrinsics_per_account, account_num),
+		|b| {
+			b.iter_batched(
+				|| {
+					let prepare_extrinsics = create_account_extrinsics(&*dave.client, &accounts);
+
+					benchmark_handle.block_on(future::join_all(
+						prepare_extrinsics.into_iter().map(|tx| {
+							submit_tx_and_wait_for_inclusion(
+								&dave.transaction_pool,
+								tx,
+								&*dave.client,
+								true,
+							)
+						}),
+					));
+
+					create_benchmark_extrinsics(&*dave.client, &accounts, extrinsics_per_account)
+				},
+				|extrinsics| {
+					benchmark_handle.block_on(future::join_all(extrinsics.into_iter().map(|tx| {
+						submit_tx_and_wait_for_inclusion(
+							&dave.transaction_pool,
+							tx,
+							&*dave.client,
+							false,
+						)
+					})));
+
+					println!("Finished {}", counter);
+					counter += 1;
+				},
+				BatchSize::SmallInput,
+			)
+		},
+	);
+
+	runtime.block_on(async {
+		join!(
+			alice.task_manager.clean_shutdown(),
+			bob.task_manager.clean_shutdown(),
+			charlie.task_manager.clean_shutdown(),
+			dave.task_manager.clean_shutdown(),
+		)
+	});
+}
+
+criterion_group!(benches, transaction_throughput_benchmarks);
+criterion_main!(benches);
@@ -29,7 +29,9 @@ use cumulus_client_service::{
 };
 use cumulus_primitives_core::ParaId;
 use cumulus_test_runtime::{Hash, Header, NodeBlock as Block, RuntimeApi};
+use frame_system_rpc_runtime_api::AccountNonceApi;
 use polkadot_primitives::v1::{CollatorPair, Hash as PHash, PersistedValidationData};
+use polkadot_service::ProvideRuntimeApi;
 use sc_client_api::execution_extensions::ExecutionStrategies;
 use sc_network::{config::TransportConfig, multiaddr, NetworkService};
 use sc_service::{
@@ -98,6 +100,9 @@ pub type Client = TFullClient<
 	sc_executor::NativeElseWasmExecutor<RuntimeExecutor>,
 >;

+/// Transaction pool type used by the test service
+pub type TransactionPool = Arc<sc_transaction_pool::FullPool<Block, Client>>;
+
 /// Starts a `ServiceBuilder` for a full service.
 ///
 /// Use this macro if you don't actually need the full service, but just the builder in order to
@@ -174,6 +179,7 @@ async fn start_node_impl<RB>(
 	Arc<Client>,
 	Arc<NetworkService<Block, H256>>,
 	RpcHandlers,
+	TransactionPool,
 )>
 where
 	RB: Fn(Arc<Client>) -> Result<jsonrpc_core::IoHandler<sc_rpc::Metadata>, sc_service::Error>
@@ -266,7 +272,7 @@ where
 				let proposer_factory = sc_basic_authorship::ProposerFactory::with_proof_recording(
 					task_manager.spawn_handle(),
 					client.clone(),
-					transaction_pool,
+					transaction_pool.clone(),
 					prometheus_registry.as_ref(),
 					None,
 				);
@@ -345,7 +351,7 @@ where

 	start_network.start_network();

-	Ok((task_manager, client, network, rpc_handlers))
+	Ok((task_manager, client, network, rpc_handlers, transaction_pool))
 }

 /// A Cumulus test node instance used for testing.
@@ -361,6 +367,8 @@ pub struct TestNode {
 	pub addr: MultiaddrWithPeerId,
 	/// RPCHandlers to make RPC queries.
 	pub rpc_handlers: RpcHandlers,
+	/// Node's transaction pool
+	pub transaction_pool: TransactionPool,
 }

 enum Consensus {
@@ -519,7 +527,7 @@ impl TestNodeBuilder {
 			format!("{} (relay chain)", relay_chain_config.network.node_name);

 		let multiaddr = parachain_config.network.listen_addresses[0].clone();
-		let (task_manager, client, network, rpc_handlers) = start_node_impl(
+		let (task_manager, client, network, rpc_handlers, transaction_pool) = start_node_impl(
 			parachain_config,
 			self.collator_key,
 			relay_chain_config,
@@ -534,7 +542,7 @@ impl TestNodeBuilder {
 		let peer_id = network.local_peer_id().clone();
 		let addr = MultiaddrWithPeerId { multiaddr, peer_id };

-		TestNode { task_manager, client, network, addr, rpc_handlers }
+		TestNode { task_manager, client, network, addr, rpc_handlers, transaction_pool }
 	}
 }

@@ -651,7 +659,7 @@ impl TestNode {
 		function: impl Into<runtime::Call>,
 		caller: Sr25519Keyring,
 	) -> Result<RpcTransactionOutput, RpcTransactionError> {
-		let extrinsic = construct_extrinsic(&*self.client, function, caller);
+		let extrinsic = construct_extrinsic(&*self.client, function, caller.pair(), Some(0));

 		self.rpc_handlers.send_transaction(extrinsic.into()).await
 	}
@@ -669,17 +677,27 @@ impl TestNode {
 	}
 }

+/// Fetch account nonce for key pair
+pub fn fetch_nonce(client: &Client, account: sp_core::sr25519::Public) -> u32 {
+	let best_hash = client.chain_info().best_hash;
+	client
+		.runtime_api()
+		.account_nonce(&generic::BlockId::Hash(best_hash), account.into())
+		.expect("Fetching account nonce works; qed")
+}
+
 /// Construct an extrinsic that can be applied to the test runtime.
 pub fn construct_extrinsic(
 	client: &Client,
 	function: impl Into<runtime::Call>,
-	caller: Sr25519Keyring,
+	caller: sp_core::sr25519::Pair,
+	nonce: Option<u32>,
 ) -> runtime::UncheckedExtrinsic {
 	let function = function.into();
 	let current_block_hash = client.info().best_hash;
 	let current_block = client.info().best_number.saturated_into();
 	let genesis_block = client.hash(0).unwrap().unwrap();
-	let nonce = 0;
+	let nonce = nonce.unwrap_or_else(|| fetch_nonce(client, caller.public()));
 	let period = runtime::BlockHashCount::get()
 		.checked_next_power_of_two()
 		.map(|c| c / 2)