Add benchmarks for for import/validation/production (#2579)

* Add benchmarks

* Remove endowed accounts

* Fix group

* Remove duplicate code

* Comments

* Use execute_block instead of block import

* Apply suggestions from code review

Co-authored-by: Anton <anton.kalyaev@gmail.com>

* Add missing comments

* Clippy

* Reorganize utils

* Remove unused import

* Switch to compiled again

* Apply suggestions from code review

Co-authored-by: Davide Galassi <davxy@datawok.net>

* Refactor local_testnet

* Do not write wasm blob to file

* Remove unnecessary block

* Add comment fixes

* fmt

* lock

---------

Co-authored-by: Anton <anton.kalyaev@gmail.com>
Co-authored-by: Davide Galassi <davxy@datawok.net>
This commit is contained in:
Sebastian Kunert
2023-06-05 16:41:20 +02:00
committed by GitHub
parent 005b9d0989
commit c047ec900a
14 changed files with 1711 additions and 41 deletions
+41
View File
@@ -19,6 +19,7 @@ serde = { version = "1.0.163", features = ["derive"] }
tokio = { version = "1.28.2", features = ["macros"] }
tracing = "0.1.37"
url = "2.3.1"
tempfile = "3.5.0"
# Substrate
frame-system = { git = "https://github.com/paritytech/substrate", branch = "master" }
@@ -39,14 +40,19 @@ sp-arithmetic = { git = "https://github.com/paritytech/substrate", branch = "mas
sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-io = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-runtime = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master" }
sp-state-machine = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-timestamp = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-trie = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-consensus = { git = "https://github.com/paritytech/substrate", branch = "master" }
substrate-test-client = { git = "https://github.com/paritytech/substrate", branch = "master" }
sc-cli = { git = "https://github.com/paritytech/substrate", branch = "master" }
sc-block-builder = { git = "https://github.com/paritytech/substrate", branch = "master" }
sc-executor-wasmtime = { git = "https://github.com/paritytech/substrate", branch = "master" }
sc-executor-common = { git = "https://github.com/paritytech/substrate", branch = "master" }
# Polkadot
polkadot-primitives = { git = "https://github.com/paritytech/polkadot", branch = "master" }
@@ -70,10 +76,21 @@ cumulus-test-relay-validation-worker-provider = { path = "../relay-validation-wo
cumulus-test-runtime = { path = "../runtime" }
cumulus-relay-chain-minimal-node = { path = "../../client/relay-chain-minimal-node" }
cumulus-client-pov-recovery = { path = "../../client/pov-recovery" }
cumulus-test-relay-sproof-builder = { path = "../relay-sproof-builder" }
cumulus-pallet-parachain-system = { path = "../../pallets/parachain-system", default-features = false }
pallet-timestamp = { git = "https://github.com/paritytech/substrate", branch = "master" }
[dev-dependencies]
futures = "0.3.28"
portpicker = "0.1.1"
kitchensink-runtime = { git = "https://github.com/paritytech/substrate", branch = "master" }
rococo-parachain-runtime = { path = "../../parachains/runtimes/testing/rococo-parachain" }
node-cli = { git = "https://github.com/paritytech/substrate", branch = "master" }
pallet-im-online = { git = "https://github.com/paritytech/substrate", branch = "master" }
node-primitives = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-consensus-grandpa = { git = "https://github.com/paritytech/substrate", branch = "master" }
sp-authority-discovery = { git = "https://github.com/paritytech/substrate", branch = "master" }
cumulus-test-client = { path = "../client" }
# Polkadot dependencies
polkadot-test-service = { git = "https://github.com/paritytech/polkadot", branch = "master" }
@@ -88,3 +105,27 @@ runtime-benchmarks = ["polkadot-test-service/runtime-benchmarks"]
[[bench]]
name = "transaction_throughput"
harness = false
[[bench]]
name = "block_import"
harness = false
[[bench]]
name = "block_production"
harness = false
[[bench]]
name = "block_production_glutton"
harness = false
[[bench]]
name = "block_import_glutton"
harness = false
[[bench]]
name = "validate_block"
harness = false
[[bench]]
name = "validate_block_glutton"
harness = false
@@ -0,0 +1,79 @@
// This file is part of Cumulus.
// Copyright (C) Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput};
use sc_client_api::UsageProvider;
use core::time::Duration;
use cumulus_primitives_core::ParaId;
use sc_block_builder::{BlockBuilderProvider, RecordProof};
use sp_api::{Core, ProvideRuntimeApi};
use sp_keyring::Sr25519Keyring::Alice;
use cumulus_test_service::bench_utils as utils;
fn benchmark_block_import(c: &mut Criterion) {
sp_tracing::try_init_simple();
let runtime = tokio::runtime::Runtime::new().expect("creating tokio runtime doesn't fail; qed");
let para_id = ParaId::from(100);
let tokio_handle = runtime.handle();
// Create enough accounts to fill the block with transactions.
// Each account should only be included in one transfer.
let (src_accounts, dst_accounts, account_ids) = utils::create_benchmark_accounts();
let alice = runtime.block_on(
cumulus_test_service::TestNodeBuilder::new(para_id, tokio_handle.clone(), Alice)
// Preload all accounts with funds for the transfers
.endowed_accounts(account_ids)
.build(),
);
let client = alice.client;
let (max_transfer_count, extrinsics) =
utils::create_benchmarking_transfer_extrinsics(&client, &src_accounts, &dst_accounts);
let parent_hash = client.usage_info().chain.best_hash;
let mut block_builder =
client.new_block_at(parent_hash, Default::default(), RecordProof::No).unwrap();
for extrinsic in extrinsics {
block_builder.push(extrinsic).unwrap();
}
let benchmark_block = block_builder.build().unwrap();
let mut group = c.benchmark_group("Block import");
group.sample_size(20);
group.measurement_time(Duration::from_secs(120));
group.throughput(Throughput::Elements(max_transfer_count as u64));
group.bench_function(format!("(transfers = {}) block import", max_transfer_count), |b| {
b.iter_batched(
|| benchmark_block.block.clone(),
|block| {
client.runtime_api().execute_block(parent_hash, block).unwrap();
},
BatchSize::SmallInput,
)
});
}
criterion_group!(benches, benchmark_block_import);
criterion_main!(benches);
@@ -0,0 +1,93 @@
// This file is part of Cumulus.
// Copyright (C) Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
use sc_client_api::UsageProvider;
use sp_api::{Core, ProvideRuntimeApi};
use sp_arithmetic::Perbill;
use core::time::Duration;
use cumulus_primitives_core::ParaId;
use sc_block_builder::{BlockBuilderProvider, RecordProof};
use sp_keyring::Sr25519Keyring::Alice;
use cumulus_test_service::bench_utils as utils;
fn benchmark_block_import(c: &mut Criterion) {
sp_tracing::try_init_simple();
let runtime = tokio::runtime::Runtime::new().expect("creating tokio runtime doesn't fail; qed");
let para_id = ParaId::from(100);
let tokio_handle = runtime.handle();
let alice = runtime.block_on(
cumulus_test_service::TestNodeBuilder::new(para_id, tokio_handle.clone(), Alice).build(),
);
let client = alice.client;
let mut group = c.benchmark_group("Block import");
group.sample_size(20);
group.measurement_time(Duration::from_secs(120));
let mut initialize_glutton_pallet = true;
for (compute_percent, storage_percent) in &[
(Perbill::from_percent(100), Perbill::from_percent(0)),
(Perbill::from_percent(100), Perbill::from_percent(100)),
] {
let block = utils::set_glutton_parameters(
&client,
initialize_glutton_pallet,
compute_percent,
storage_percent,
);
initialize_glutton_pallet = false;
runtime.block_on(utils::import_block(&client, &block, false));
// Build the block we will use for benchmarking
let parent_hash = client.usage_info().chain.best_hash;
let parent_header = client.header(parent_hash).expect("Just fetched this hash.").unwrap();
let mut block_builder =
client.new_block_at(parent_hash, Default::default(), RecordProof::No).unwrap();
block_builder
.push(utils::extrinsic_set_validation_data(parent_header.clone()).clone())
.unwrap();
block_builder.push(utils::extrinsic_set_time(&client)).unwrap();
let benchmark_block = block_builder.build().unwrap();
group.bench_function(
format!(
"(compute = {:?}, storage = {:?}) block import",
compute_percent, storage_percent
),
|b| {
b.iter_batched(
|| benchmark_block.block.clone(),
|block| {
client.runtime_api().execute_block(parent_hash, block).unwrap();
},
BatchSize::SmallInput,
)
},
);
}
}
criterion_group!(benches, benchmark_block_import);
criterion_main!(benches);
@@ -0,0 +1,111 @@
// This file is part of Cumulus.
// Copyright (C) Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput};
use sc_client_api::UsageProvider;
use core::time::Duration;
use cumulus_primitives_core::ParaId;
use sc_block_builder::{BlockBuilderProvider, RecordProof};
use sp_keyring::Sr25519Keyring::Alice;
use cumulus_test_service::bench_utils as utils;
fn benchmark_block_production(c: &mut Criterion) {
sp_tracing::try_init_simple();
let runtime = tokio::runtime::Runtime::new().expect("creating tokio runtime doesn't fail; qed");
let tokio_handle = runtime.handle();
// Create enough accounts to fill the block with transactions.
// Each account should only be included in one transfer.
let (src_accounts, dst_accounts, account_ids) = utils::create_benchmark_accounts();
let para_id = ParaId::from(100);
let alice = runtime.block_on(
cumulus_test_service::TestNodeBuilder::new(para_id, tokio_handle.clone(), Alice)
// Preload all accounts with funds for the transfers
.endowed_accounts(account_ids)
.build(),
);
let client = alice.client;
let parent_hash = client.usage_info().chain.best_hash;
let parent_header = client.header(parent_hash).expect("Just fetched this hash.").unwrap();
let set_validation_data_extrinsic = utils::extrinsic_set_validation_data(parent_header);
let mut block_builder = client.new_block(Default::default()).unwrap();
block_builder.push(utils::extrinsic_set_time(&client)).unwrap();
block_builder.push(set_validation_data_extrinsic).unwrap();
let built_block = block_builder.build().unwrap();
runtime.block_on(utils::import_block(&client, &built_block.block, false));
let (max_transfer_count, extrinsics) =
utils::create_benchmarking_transfer_extrinsics(&client, &src_accounts, &dst_accounts);
let mut group = c.benchmark_group("Block production");
group.sample_size(20);
group.measurement_time(Duration::from_secs(120));
group.throughput(Throughput::Elements(max_transfer_count as u64));
let best_hash = client.chain_info().best_hash;
group.bench_function(
format!("(proof = true, transfers = {}) block production", max_transfer_count),
|b| {
b.iter_batched(
|| extrinsics.clone(),
|extrinsics| {
let mut block_builder = client
.new_block_at(best_hash, Default::default(), RecordProof::Yes)
.unwrap();
for extrinsic in extrinsics {
block_builder.push(extrinsic).unwrap();
}
block_builder.build().unwrap()
},
BatchSize::SmallInput,
)
},
);
group.bench_function(
format!("(proof = false, transfers = {}) block production", max_transfer_count),
|b| {
b.iter_batched(
|| extrinsics.clone(),
|extrinsics| {
let mut block_builder = client
.new_block_at(best_hash, Default::default(), RecordProof::No)
.unwrap();
for extrinsic in extrinsics {
block_builder.push(extrinsic).unwrap();
}
block_builder.build().unwrap()
},
BatchSize::SmallInput,
)
},
);
}
criterion_group!(benches, benchmark_block_production);
criterion_main!(benches);
@@ -0,0 +1,114 @@
// This file is part of Cumulus.
// Copyright (C) Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
use sc_client_api::UsageProvider;
use sp_arithmetic::Perbill;
use core::time::Duration;
use cumulus_primitives_core::ParaId;
use sc_block_builder::{BlockBuilderProvider, RecordProof};
use sp_keyring::Sr25519Keyring::Alice;
use cumulus_test_service::bench_utils as utils;
fn benchmark_block_production_compute(c: &mut Criterion) {
sp_tracing::try_init_simple();
let runtime = tokio::runtime::Runtime::new().expect("creating tokio runtime doesn't fail; qed");
let tokio_handle = runtime.handle();
let para_id = ParaId::from(100);
let alice = runtime.block_on(
cumulus_test_service::TestNodeBuilder::new(para_id, tokio_handle.clone(), Alice).build(),
);
let client = alice.client;
let mut group = c.benchmark_group("Block production");
group.sample_size(20);
group.measurement_time(Duration::from_secs(120));
let mut initialize_glutton_pallet = true;
for (compute_level, storage_level) in &[
(Perbill::from_percent(100), Perbill::from_percent(0)),
(Perbill::from_percent(100), Perbill::from_percent(100)),
] {
let block = utils::set_glutton_parameters(
&client,
initialize_glutton_pallet,
compute_level,
storage_level,
);
runtime.block_on(utils::import_block(&client, &block, false));
initialize_glutton_pallet = false;
let parent_hash = client.usage_info().chain.best_hash;
let parent_header = client.header(parent_hash).expect("Just fetched this hash.").unwrap();
let set_validation_data_extrinsic = utils::extrinsic_set_validation_data(parent_header);
let set_time_extrinsic = utils::extrinsic_set_time(&client);
let best_hash = client.chain_info().best_hash;
group.bench_function(
format!(
"(compute = {:?}, storage = {:?}, proof = true) block production",
compute_level, storage_level
),
|b| {
b.iter_batched(
|| (set_validation_data_extrinsic.clone(), set_time_extrinsic.clone()),
|(validation_data, time)| {
let mut block_builder = client
.new_block_at(best_hash, Default::default(), RecordProof::Yes)
.unwrap();
block_builder.push(validation_data).unwrap();
block_builder.push(time).unwrap();
block_builder.build().unwrap()
},
BatchSize::SmallInput,
)
},
);
group.bench_function(
format!(
"(compute = {:?}, storage = {:?}, proof = false) block production",
compute_level, storage_level
),
|b| {
b.iter_batched(
|| (set_validation_data_extrinsic.clone(), set_time_extrinsic.clone()),
|(validation_data, time)| {
let mut block_builder = client
.new_block_at(best_hash, Default::default(), RecordProof::No)
.unwrap();
block_builder.push(validation_data).unwrap();
block_builder.push(time).unwrap();
block_builder.build().unwrap()
},
BatchSize::SmallInput,
)
},
);
}
}
criterion_group!(benches, benchmark_block_production_compute);
criterion_main!(benches);
@@ -0,0 +1,163 @@
// This file is part of Cumulus.
// Copyright (C) Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use codec::{Decode, Encode};
use core::time::Duration;
use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput};
use cumulus_primitives_core::{relay_chain::AccountId, PersistedValidationData, ValidationParams};
use cumulus_test_client::{
generate_extrinsic_with_pair, BuildParachainBlockData, InitBlockBuilder, TestClientBuilder,
ValidationResult,
};
use cumulus_test_relay_sproof_builder::RelayStateSproofBuilder;
use cumulus_test_runtime::{BalancesCall, Block, Header, UncheckedExtrinsic};
use cumulus_test_service::bench_utils as utils;
use polkadot_primitives::HeadData;
use sc_block_builder::BlockBuilderProvider;
use sc_client_api::UsageProvider;
use sc_executor_common::wasm_runtime::WasmModule;
use sp_blockchain::{ApplyExtrinsicFailed::Validity, Error::ApplyExtrinsicFailed};
use sp_core::{sr25519, Pair};
use sp_runtime::{
traits::Header as HeaderT,
transaction_validity::{InvalidTransaction, TransactionValidityError},
};
fn create_extrinsics(
client: &cumulus_test_client::Client,
src_accounts: &[sr25519::Pair],
dst_accounts: &[sr25519::Pair],
) -> (usize, Vec<UncheckedExtrinsic>) {
// Add as many tranfer extrinsics as possible into a single block.
let mut block_builder = client.new_block(Default::default()).unwrap();
let mut max_transfer_count = 0;
let mut extrinsics = Vec::new();
for (src, dst) in src_accounts.iter().zip(dst_accounts.iter()) {
let extrinsic: UncheckedExtrinsic = generate_extrinsic_with_pair(
client,
src.clone(),
BalancesCall::transfer_keep_alive { dest: AccountId::from(dst.public()), value: 10000 },
None,
);
match block_builder.push(extrinsic.clone()) {
Ok(_) => {},
Err(ApplyExtrinsicFailed(Validity(TransactionValidityError::Invalid(
InvalidTransaction::ExhaustsResources,
)))) => break,
Err(error) => panic!("{}", error),
}
extrinsics.push(extrinsic);
max_transfer_count += 1;
}
(max_transfer_count, extrinsics)
}
fn benchmark_block_validation(c: &mut Criterion) {
sp_tracing::try_init_simple();
// Create enough accounts to fill the block with transactions.
// Each account should only be included in one transfer.
let (src_accounts, dst_accounts, account_ids) = utils::create_benchmark_accounts();
let mut test_client_builder = TestClientBuilder::with_default_backend()
.set_execution_strategy(sc_client_api::ExecutionStrategy::AlwaysWasm);
let genesis_init = test_client_builder.genesis_init_mut();
*genesis_init = cumulus_test_client::GenesisParameters { endowed_accounts: account_ids };
let client = test_client_builder.build_with_native_executor(None).0;
let (max_transfer_count, extrinsics) = create_extrinsics(&client, &src_accounts, &dst_accounts);
let parent_hash = client.usage_info().chain.best_hash;
let parent_header = client.header(parent_hash).expect("Just fetched this hash.").unwrap();
let validation_data = PersistedValidationData {
relay_parent_number: 1,
parent_head: parent_header.encode().into(),
..Default::default()
};
let mut block_builder = client.init_block_builder(Some(validation_data), Default::default());
for extrinsic in extrinsics {
block_builder.push(extrinsic).unwrap();
}
let parachain_block = block_builder.build_parachain_block(*parent_header.state_root());
let proof_size_in_kb = parachain_block.storage_proof().encode().len() as f64 / 1024f64;
let runtime = utils::get_wasm_module();
let sproof_builder: RelayStateSproofBuilder = Default::default();
let (relay_parent_storage_root, _) = sproof_builder.into_state_root_and_proof();
let encoded_params = ValidationParams {
block_data: cumulus_test_client::BlockData(parachain_block.encode()),
parent_head: HeadData(parent_header.encode()),
relay_parent_number: 1,
relay_parent_storage_root,
}
.encode();
// This is not strictly necessary for this benchmark, but
// let us make sure that the result of `validate_block` is what
// we expect.
verify_expected_result(&runtime, &encoded_params, parachain_block.into_block());
let mut group = c.benchmark_group("Block validation");
group.sample_size(20);
group.measurement_time(Duration::from_secs(120));
group.throughput(Throughput::Elements(max_transfer_count as u64));
group.bench_function(
format!(
"(transfers = {}, proof_size = {}kb) block validation",
max_transfer_count, proof_size_in_kb
),
|b| {
b.iter_batched(
|| runtime.new_instance().unwrap(),
|mut instance| {
instance.call_export("validate_block", &encoded_params).unwrap();
},
BatchSize::SmallInput,
)
},
);
}
fn verify_expected_result(
runtime: &Box<dyn WasmModule>,
encoded_params: &[u8],
parachain_block: Block,
) {
let res = runtime
.new_instance()
.unwrap()
.call_export("validate_block", encoded_params)
.expect("Call `validate_block`.");
let validation_result =
ValidationResult::decode(&mut &res[..]).expect("Decode `ValidationResult`.");
let header =
Header::decode(&mut &validation_result.head_data.0[..]).expect("Decodes `Header`.");
assert_eq!(parachain_block.header, header);
}
criterion_group!(benches, benchmark_block_validation);
criterion_main!(benches);
@@ -0,0 +1,211 @@
// This file is part of Cumulus.
// Copyright (C) Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use codec::{Decode, Encode};
use core::time::Duration;
use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
use cumulus_primitives_core::{relay_chain::AccountId, PersistedValidationData, ValidationParams};
use cumulus_test_client::{
generate_extrinsic_with_pair, BuildParachainBlockData, Client, InitBlockBuilder,
ParachainBlockData, TestClientBuilder, ValidationResult,
};
use cumulus_test_relay_sproof_builder::RelayStateSproofBuilder;
use cumulus_test_runtime::{Block, GluttonCall, Header, SudoCall};
use polkadot_primitives::HeadData;
use sc_client_api::UsageProvider;
use sc_consensus::{BlockImport, BlockImportParams, ForkChoiceStrategy, ImportResult, StateAction};
use sc_executor_common::wasm_runtime::WasmModule;
use sp_api::ProvideRuntimeApi;
use frame_system_rpc_runtime_api::AccountNonceApi;
use sp_arithmetic::Perbill;
use sp_consensus::BlockOrigin;
use sp_keyring::Sr25519Keyring::Alice;
use sp_runtime::traits::Header as HeaderT;
use cumulus_test_service::bench_utils as utils;
async fn import_block(
mut client: &cumulus_test_client::Client,
built: cumulus_test_runtime::Block,
import_existing: bool,
) {
let mut params = BlockImportParams::new(BlockOrigin::File, built.header.clone());
params.body = Some(built.extrinsics.clone());
params.state_action = StateAction::Execute;
params.fork_choice = Some(ForkChoiceStrategy::LongestChain);
params.import_existing = import_existing;
let import_result = client.import_block(params).await;
assert!(matches!(import_result, Ok(ImportResult::Imported(_))));
}
fn benchmark_block_validation(c: &mut Criterion) {
sp_tracing::try_init_simple();
let runtime = tokio::runtime::Runtime::new().expect("creating tokio runtime doesn't fail; qed");
let endowed_accounts = vec![AccountId::from(Alice.public())];
let mut test_client_builder = TestClientBuilder::with_default_backend()
.set_execution_strategy(sc_client_api::ExecutionStrategy::NativeElseWasm);
let genesis_init = test_client_builder.genesis_init_mut();
*genesis_init = cumulus_test_client::GenesisParameters { endowed_accounts };
let client = test_client_builder.build_with_native_executor(None).0;
let mut group = c.benchmark_group("Block validation");
group.sample_size(20);
group.measurement_time(Duration::from_secs(120));
// In the first iteration we want to initialie the glutton pallet
let mut is_first = true;
for (compute_percent, storage_percent) in &[
(Perbill::from_percent(100), Perbill::from_percent(0)),
(Perbill::from_percent(100), Perbill::from_percent(100)),
] {
let parachain_block =
set_glutton_parameters(&client, is_first, compute_percent, storage_percent);
is_first = false;
runtime.block_on(import_block(&client, parachain_block.clone().into_block(), false));
// Build benchmark block
let parent_hash = client.usage_info().chain.best_hash;
let parent_header = client.header(parent_hash).expect("Just fetched this hash.").unwrap();
let validation_data = PersistedValidationData {
relay_parent_number: 1,
parent_head: parent_header.encode().into(),
..Default::default()
};
let block_builder = client.init_block_builder(Some(validation_data), Default::default());
let parachain_block = block_builder.build_parachain_block(*parent_header.state_root());
let proof_size_in_kb = parachain_block.storage_proof().encode().len() as f64 / 1024f64;
runtime.block_on(import_block(&client, parachain_block.clone().into_block(), false));
let runtime = utils::get_wasm_module();
let sproof_builder: RelayStateSproofBuilder = Default::default();
let (relay_parent_storage_root, _) = sproof_builder.clone().into_state_root_and_proof();
let encoded_params = ValidationParams {
block_data: cumulus_test_client::BlockData(parachain_block.clone().encode()),
parent_head: HeadData(parent_header.encode()),
relay_parent_number: 1,
relay_parent_storage_root,
}
.encode();
// This is not strictly necessary for this benchmark, but
// let us make sure that the result of `validate_block` is what
// we expect.
verify_expected_result(&runtime, &encoded_params, parachain_block.into_block());
group.bench_function(
format!(
"(compute = {:?}, storage = {:?}, proof_size = {}kb) block validation",
compute_percent, storage_percent, proof_size_in_kb
),
|b| {
b.iter_batched(
|| runtime.new_instance().unwrap(),
|mut instance| {
instance.call_export("validate_block", &encoded_params).unwrap();
},
BatchSize::SmallInput,
)
},
);
}
}
fn verify_expected_result(runtime: &Box<dyn WasmModule>, encoded_params: &[u8], block: Block) {
let res = runtime
.new_instance()
.unwrap()
.call_export("validate_block", encoded_params)
.expect("Call `validate_block`.");
let validation_result =
ValidationResult::decode(&mut &res[..]).expect("Decode `ValidationResult`.");
let header =
Header::decode(&mut &validation_result.head_data.0[..]).expect("Decodes `Header`.");
assert_eq!(block.header, header);
}
fn set_glutton_parameters(
client: &Client,
initialize: bool,
compute_percent: &Perbill,
storage_percent: &Perbill,
) -> ParachainBlockData {
let parent_hash = client.usage_info().chain.best_hash;
let parent_header = client.header(parent_hash).expect("Just fetched this hash.").unwrap();
let mut last_nonce = client
.runtime_api()
.account_nonce(parent_hash, Alice.into())
.expect("Fetching account nonce works; qed");
let validation_data = PersistedValidationData {
relay_parent_number: 1,
parent_head: parent_header.encode().into(),
..Default::default()
};
let mut extrinsics = vec![];
if initialize {
extrinsics.push(generate_extrinsic_with_pair(
client,
Alice.into(),
SudoCall::sudo {
call: Box::new(
GluttonCall::initialize_pallet { new_count: 5000, witness_count: None }.into(),
),
},
Some(last_nonce),
));
last_nonce += 1;
}
let set_compute = generate_extrinsic_with_pair(
client,
Alice.into(),
SudoCall::sudo {
call: Box::new(GluttonCall::set_compute { compute: *compute_percent }.into()),
},
Some(last_nonce),
);
last_nonce += 1;
extrinsics.push(set_compute);
let set_storage = generate_extrinsic_with_pair(
client,
Alice.into(),
SudoCall::sudo {
call: Box::new(GluttonCall::set_storage { storage: *storage_percent }.into()),
},
Some(last_nonce),
);
extrinsics.push(set_storage);
let mut block_builder = client.init_block_builder(Some(validation_data), Default::default());
for extrinsic in extrinsics {
block_builder.push(extrinsic).unwrap();
}
block_builder.build_parachain_block(*parent_header.state_root())
}
criterion_group!(benches, benchmark_block_validation);
criterion_main!(benches);
+262
View File
@@ -0,0 +1,262 @@
// This file is part of Cumulus.
// Copyright (C) Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use codec::Encode;
use crate::{construct_extrinsic, Client as TestClient};
use cumulus_primitives_core::{relay_chain::AccountId, PersistedValidationData};
use cumulus_primitives_parachain_inherent::ParachainInherentData;
use cumulus_test_relay_sproof_builder::RelayStateSproofBuilder;
use cumulus_test_runtime::{
BalancesCall, GluttonCall, NodeBlock, SudoCall, UncheckedExtrinsic, WASM_BINARY,
};
use frame_system_rpc_runtime_api::AccountNonceApi;
use polkadot_primitives::HeadData;
use sc_block_builder::BlockBuilderProvider;
use sc_client_api::UsageProvider;
use sc_consensus::{
block_import::{BlockImportParams, ForkChoiceStrategy},
BlockImport, ImportResult, StateAction,
};
use sc_executor::DEFAULT_HEAP_ALLOC_STRATEGY;
use sc_executor_common::runtime_blob::RuntimeBlob;
use sp_api::ProvideRuntimeApi;
use sp_arithmetic::Perbill;
use sp_blockchain::{ApplyExtrinsicFailed::Validity, Error::ApplyExtrinsicFailed};
use sp_consensus::BlockOrigin;
use sp_core::{sr25519, Pair};
use sp_keyring::Sr25519Keyring::Alice;
use sp_runtime::{
transaction_validity::{InvalidTransaction, TransactionValidityError},
AccountId32, OpaqueExtrinsic,
};
/// Accounts to use for transfer transactions. Enough for 5000 transactions.
const NUM_ACCOUNTS: usize = 10000;
/// Create accounts by deriving from Alice
pub fn create_benchmark_accounts() -> (Vec<sr25519::Pair>, Vec<sr25519::Pair>, Vec<AccountId32>) {
let accounts: Vec<sr25519::Pair> = (0..NUM_ACCOUNTS)
.map(|idx| {
Pair::from_string(&format!("{}/{}", Alice.to_seed(), idx), None)
.expect("Creates account pair")
})
.collect();
let account_ids = accounts
.iter()
.map(|account| AccountId::from(account.public()))
.collect::<Vec<AccountId>>();
let (src_accounts, dst_accounts) = accounts.split_at(NUM_ACCOUNTS / 2);
(src_accounts.to_vec(), dst_accounts.to_vec(), account_ids)
}
/// Create a timestamp extrinsic ahead by `MinimumPeriod` of the last known timestamp
pub fn extrinsic_set_time(client: &TestClient) -> OpaqueExtrinsic {
let best_number = client.usage_info().chain.best_number;
let timestamp = best_number as u64 * cumulus_test_runtime::MinimumPeriod::get();
cumulus_test_runtime::UncheckedExtrinsic {
signature: None,
function: cumulus_test_runtime::RuntimeCall::Timestamp(pallet_timestamp::Call::set {
now: timestamp,
}),
}
.into()
}
/// Create a set validation data extrinsic
pub fn extrinsic_set_validation_data(
parent_header: cumulus_test_runtime::Header,
) -> OpaqueExtrinsic {
let sproof_builder = RelayStateSproofBuilder { para_id: 100.into(), ..Default::default() };
let parent_head = HeadData(parent_header.encode());
let (relay_parent_storage_root, relay_chain_state) = sproof_builder.into_state_root_and_proof();
let data = ParachainInherentData {
validation_data: PersistedValidationData {
parent_head,
relay_parent_number: 10,
relay_parent_storage_root,
max_pov_size: 10000,
},
relay_chain_state,
downward_messages: Default::default(),
horizontal_messages: Default::default(),
};
cumulus_test_runtime::UncheckedExtrinsic {
signature: None,
function: cumulus_test_runtime::RuntimeCall::ParachainSystem(
cumulus_pallet_parachain_system::Call::set_validation_data { data },
),
}
.into()
}
/// Import block into the given client and make sure the import was successful
pub async fn import_block(mut client: &TestClient, block: &NodeBlock, import_existing: bool) {
let mut params = BlockImportParams::new(BlockOrigin::File, block.header.clone());
params.body = Some(block.extrinsics.clone());
params.state_action = StateAction::Execute;
params.fork_choice = Some(ForkChoiceStrategy::LongestChain);
params.import_existing = import_existing;
let import_result = client.import_block(params).await;
assert!(
matches!(import_result, Ok(ImportResult::Imported(_))),
"Unexpected block import result: {:?}!",
import_result
);
}
/// Creates transfer extrinsics pair-wise from elements of `src_accounts` to `dst_accounts`.
pub fn create_benchmarking_transfer_extrinsics(
client: &TestClient,
src_accounts: &[sr25519::Pair],
dst_accounts: &[sr25519::Pair],
) -> (usize, Vec<OpaqueExtrinsic>) {
// Add as many tranfer extrinsics as possible into a single block.
let mut block_builder = client.new_block(Default::default()).unwrap();
let mut max_transfer_count = 0;
let mut extrinsics = Vec::new();
// Every block needs one timestamp extrinsic.
let time_ext = extrinsic_set_time(client);
extrinsics.push(time_ext);
// Every block needs tone set_validation_data extrinsic.
let parent_hash = client.usage_info().chain.best_hash;
let parent_header = client.header(parent_hash).expect("Just fetched this hash.").unwrap();
let set_validation_data_extrinsic = extrinsic_set_validation_data(parent_header);
extrinsics.push(set_validation_data_extrinsic);
for (src, dst) in src_accounts.iter().zip(dst_accounts.iter()) {
let extrinsic: UncheckedExtrinsic = construct_extrinsic(
client,
BalancesCall::transfer_keep_alive { dest: AccountId::from(dst.public()), value: 10000 },
src.clone(),
Some(0),
);
match block_builder.push(extrinsic.clone().into()) {
Ok(_) => {},
Err(ApplyExtrinsicFailed(Validity(TransactionValidityError::Invalid(
InvalidTransaction::ExhaustsResources,
)))) => break,
Err(error) => panic!("{}", error),
}
extrinsics.push(extrinsic.into());
max_transfer_count += 1;
}
if max_transfer_count >= src_accounts.len() {
panic!("Block could fit more transfers, increase NUM_ACCOUNTS to generate more accounts.");
}
(max_transfer_count, extrinsics)
}
/// Prepare cumulus test runtime for execution
pub fn get_wasm_module() -> Box<dyn sc_executor_common::wasm_runtime::WasmModule> {
let blob = RuntimeBlob::uncompress_if_needed(
WASM_BINARY.expect("You need to build the WASM binaries to run the benchmark!"),
)
.unwrap();
let config = sc_executor_wasmtime::Config {
allow_missing_func_imports: true,
cache_path: None,
semantics: sc_executor_wasmtime::Semantics {
heap_alloc_strategy: DEFAULT_HEAP_ALLOC_STRATEGY,
instantiation_strategy: sc_executor::WasmtimeInstantiationStrategy::PoolingCopyOnWrite,
deterministic_stack_limit: None,
canonicalize_nans: false,
parallel_compilation: true,
wasm_multi_value: false,
wasm_bulk_memory: false,
wasm_reference_types: false,
wasm_simd: false,
},
};
Box::new(
sc_executor_wasmtime::create_runtime::<sp_io::SubstrateHostFunctions>(blob, config)
.expect("Unable to create wasm module."),
)
}
/// Create a block containing setup extrinsics for the glutton pallet.
pub fn set_glutton_parameters(
client: &TestClient,
initialize: bool,
compute_percent: &Perbill,
storage_percent: &Perbill,
) -> NodeBlock {
let parent_hash = client.usage_info().chain.best_hash;
let parent_header = client.header(parent_hash).expect("Just fetched this hash.").unwrap();
let mut last_nonce = client
.runtime_api()
.account_nonce(parent_hash, Alice.into())
.expect("Fetching account nonce works; qed");
let mut extrinsics = vec![];
if initialize {
// Initialize the pallet
extrinsics.push(construct_extrinsic(
client,
SudoCall::sudo {
call: Box::new(
GluttonCall::initialize_pallet { new_count: 5000, witness_count: None }.into(),
),
},
Alice.into(),
Some(last_nonce),
));
last_nonce += 1;
}
// Set compute weight that should be consumed per block
let set_compute = construct_extrinsic(
client,
SudoCall::sudo {
call: Box::new(GluttonCall::set_compute { compute: *compute_percent }.into()),
},
Alice.into(),
Some(last_nonce),
);
last_nonce += 1;
extrinsics.push(set_compute);
// Set storage weight that should be consumed per block
let set_storage = construct_extrinsic(
client,
SudoCall::sudo {
call: Box::new(GluttonCall::set_storage { storage: *storage_percent }.into()),
},
Alice.into(),
Some(last_nonce),
);
extrinsics.push(set_storage);
let mut block_builder = client.new_block(Default::default()).unwrap();
block_builder.push(extrinsic_set_time(client)).unwrap();
block_builder.push(extrinsic_set_validation_data(parent_header)).unwrap();
for extrinsic in extrinsics {
block_builder.push(extrinsic.into()).unwrap();
}
let built_block = block_builder.build().unwrap();
built_block.block
}
+41 -22
View File
@@ -80,12 +80,22 @@ where
}
/// Get the chain spec for a specific parachain ID.
pub fn get_chain_spec(id: ParaId) -> ChainSpec {
/// The given accounts are initialized with funds in addition
/// to the default known accounts.
pub fn get_chain_spec_with_extra_endowed(
id: ParaId,
extra_endowed_accounts: Vec<AccountId>,
) -> ChainSpec {
ChainSpec::from_genesis(
"Local Testnet",
"local_testnet",
ChainType::Local,
move || GenesisExt { runtime_genesis_config: local_testnet_genesis(), para_id: id },
move || GenesisExt {
runtime_genesis_config: testnet_genesis_with_default_endowed(
extra_endowed_accounts.clone(),
),
para_id: id,
},
Vec::new(),
None,
None,
@@ -95,28 +105,36 @@ pub fn get_chain_spec(id: ParaId) -> ChainSpec {
)
}
/// Local testnet genesis for testing.
pub fn local_testnet_genesis() -> cumulus_test_runtime::GenesisConfig {
testnet_genesis(
get_account_id_from_seed::<sr25519::Public>("Alice"),
vec![
get_account_id_from_seed::<sr25519::Public>("Alice"),
get_account_id_from_seed::<sr25519::Public>("Bob"),
get_account_id_from_seed::<sr25519::Public>("Charlie"),
get_account_id_from_seed::<sr25519::Public>("Dave"),
get_account_id_from_seed::<sr25519::Public>("Eve"),
get_account_id_from_seed::<sr25519::Public>("Ferdie"),
get_account_id_from_seed::<sr25519::Public>("Alice//stash"),
get_account_id_from_seed::<sr25519::Public>("Bob//stash"),
get_account_id_from_seed::<sr25519::Public>("Charlie//stash"),
get_account_id_from_seed::<sr25519::Public>("Dave//stash"),
get_account_id_from_seed::<sr25519::Public>("Eve//stash"),
get_account_id_from_seed::<sr25519::Public>("Ferdie//stash"),
],
)
/// Get the chain spec for a specific parachain ID.
pub fn get_chain_spec(id: ParaId) -> ChainSpec {
get_chain_spec_with_extra_endowed(id, Default::default())
}
fn testnet_genesis(
/// Local testnet genesis for testing.
pub fn testnet_genesis_with_default_endowed(
mut extra_endowed_accounts: Vec<AccountId>,
) -> cumulus_test_runtime::GenesisConfig {
let mut endowed = vec![
get_account_id_from_seed::<sr25519::Public>("Alice"),
get_account_id_from_seed::<sr25519::Public>("Bob"),
get_account_id_from_seed::<sr25519::Public>("Charlie"),
get_account_id_from_seed::<sr25519::Public>("Dave"),
get_account_id_from_seed::<sr25519::Public>("Eve"),
get_account_id_from_seed::<sr25519::Public>("Ferdie"),
get_account_id_from_seed::<sr25519::Public>("Alice//stash"),
get_account_id_from_seed::<sr25519::Public>("Bob//stash"),
get_account_id_from_seed::<sr25519::Public>("Charlie//stash"),
get_account_id_from_seed::<sr25519::Public>("Dave//stash"),
get_account_id_from_seed::<sr25519::Public>("Eve//stash"),
get_account_id_from_seed::<sr25519::Public>("Ferdie//stash"),
];
endowed.append(&mut extra_endowed_accounts);
testnet_genesis(get_account_id_from_seed::<sr25519::Public>("Alice"), endowed)
}
/// Creates a local testnet genesis with endowed accounts.
pub fn testnet_genesis(
root_key: AccountId,
endowed_accounts: Vec<AccountId>,
) -> cumulus_test_runtime::GenesisConfig {
@@ -126,6 +144,7 @@ fn testnet_genesis(
.expect("WASM binary was not build, please build it!")
.to_vec(),
},
glutton: Default::default(),
parachain_system: Default::default(),
balances: cumulus_test_runtime::BalancesConfig {
balances: endowed_accounts.iter().cloned().map(|k| (k, 1 << 60)).collect(),
+24 -8
View File
@@ -18,9 +18,13 @@
#![warn(missing_docs)]
/// Utilities used for benchmarking
pub mod bench_utils;
pub mod chain_spec;
mod genesis;
use runtime::AccountId;
use sc_executor::{HeapAllocStrategy, WasmExecutor, DEFAULT_HEAP_ALLOC_STRATEGY};
use std::{
future::Future,
@@ -501,6 +505,7 @@ pub struct TestNodeBuilder {
storage_update_func_relay_chain: Option<Box<dyn Fn()>>,
consensus: Consensus,
relay_chain_full_node_url: Vec<Url>,
endowed_accounts: Vec<AccountId>,
}
impl TestNodeBuilder {
@@ -523,6 +528,7 @@ impl TestNodeBuilder {
storage_update_func_relay_chain: None,
consensus: Consensus::RelayChain,
relay_chain_full_node_url: vec![],
endowed_accounts: Default::default(),
}
}
@@ -629,6 +635,12 @@ impl TestNodeBuilder {
self
}
/// Accounts which will have an initial balance.
pub fn endowed_accounts(mut self, accounts: Vec<AccountId>) -> TestNodeBuilder {
self.endowed_accounts = accounts;
self
}
/// Build the [`TestNode`].
pub async fn build(self) -> TestNode {
let parachain_config = node_config(
@@ -639,6 +651,7 @@ impl TestNodeBuilder {
self.parachain_nodes_exclusive,
self.para_id,
self.collator_key.is_some(),
self.endowed_accounts,
)
.expect("could not generate Configuration");
@@ -692,12 +705,14 @@ pub fn node_config(
nodes_exlusive: bool,
para_id: ParaId,
is_collator: bool,
endowed_accounts: Vec<AccountId>,
) -> Result<Configuration, ServiceError> {
let base_path = BasePath::new_temp_dir()?;
let root = base_path.path().join(format!("cumulus_test_service_{}", key));
let role = if is_collator { Role::Authority } else { Role::Full };
let key_seed = key.to_seed();
let mut spec = Box::new(chain_spec::get_chain_spec(para_id));
let mut spec =
Box::new(chain_spec::get_chain_spec_with_extra_endowed(para_id, endowed_accounts));
let mut storage = spec.as_storage_builder().build_storage().expect("could not build storage");
@@ -740,14 +755,15 @@ pub fn node_config(
state_pruning: Some(PruningMode::ArchiveAll),
blocks_pruning: BlocksPruning::KeepAll,
chain_spec: spec,
wasm_method: WasmExecutionMethod::default(),
// NOTE: we enforce the use of the native runtime to make the errors more debuggable
wasm_method: WasmExecutionMethod::Compiled {
instantiation_strategy: sc_executor_wasmtime::InstantiationStrategy::PoolingCopyOnWrite,
},
execution_strategies: ExecutionStrategies {
syncing: sc_client_api::ExecutionStrategy::NativeWhenPossible,
importing: sc_client_api::ExecutionStrategy::NativeWhenPossible,
block_construction: sc_client_api::ExecutionStrategy::NativeWhenPossible,
offchain_worker: sc_client_api::ExecutionStrategy::NativeWhenPossible,
other: sc_client_api::ExecutionStrategy::NativeWhenPossible,
syncing: sc_client_api::ExecutionStrategy::AlwaysWasm,
importing: sc_client_api::ExecutionStrategy::AlwaysWasm,
block_construction: sc_client_api::ExecutionStrategy::AlwaysWasm,
offchain_worker: sc_client_api::ExecutionStrategy::AlwaysWasm,
other: sc_client_api::ExecutionStrategy::AlwaysWasm,
},
rpc_addr: None,
rpc_max_connections: Default::default(),