Move cumulus zombienet tests to aura & async backing (#3568)

Cumulus test-parachain node and test runtime were still using relay
chain consensus and 12s blocktimes. With async backing around the corner
on the major chains we should switch our tests too.

Also needed to nicely test the changes coming to collators in #3168.

### Changes Overview
- Followed the [migration
guide](https://wiki.polkadot.network/docs/maintain-guides-async-backing)
for async backing for the cumulus-test-runtime
- Adjusted the cumulus-test-service to use the correct import-queue,
lookahead collator etc.
- The block validation function now uses the Aura Ext Executor so that
the seal of the block is validated
- Previous point requires that we seal block before calling into
`validate_block`, I introduced a helper function for that
- Test client adjusted to provide a slot to the relay chain proof and
the aura pre-digest

cumulus/test/client/src/block_builder.rs

@@ -15,14 +15,26 @@
 // along with Cumulus.  If not, see <http://www.gnu.org/licenses/>.
 
 use crate::Client;
+use codec::Encode;
 use cumulus_primitives_core::{ParachainBlockData, PersistedValidationData};
 use cumulus_primitives_parachain_inherent::{ParachainInherentData, INHERENT_IDENTIFIER};
 use cumulus_test_relay_sproof_builder::RelayStateSproofBuilder;
 use cumulus_test_runtime::{Block, GetLastTimestamp, Hash, Header};
 use polkadot_primitives::{BlockNumber as PBlockNumber, Hash as PHash};
-use sc_block_builder::{BlockBuilder, BlockBuilderBuilder};
+use sc_block_builder::BlockBuilderBuilder;
 use sp_api::ProvideRuntimeApi;
-use sp_runtime::traits::{Block as BlockT, Header as HeaderT};
+use sp_consensus_aura::Slot;
+use sp_runtime::{
+	traits::{Block as BlockT, Header as HeaderT},
+	Digest, DigestItem,
+};
+
+/// A struct containing a block builder and support data required to build test scenarios.
+pub struct BlockBuilderAndSupportData<'a> {
+	pub block_builder: sc_block_builder::BlockBuilder<'a, Block, Client>,
+	pub persisted_validation_data: PersistedValidationData<PHash, PBlockNumber>,
+	pub slot: Slot,
+}
 
 /// An extension for the Cumulus test client to init a block builder.
 pub trait InitBlockBuilder {
@@ -32,12 +44,15 @@ pub trait InitBlockBuilder {
 	/// valid for the test runtime.
 	///
 	/// You can use the relay chain state sproof builder to arrange required relay chain state or
-	/// just use a default one.
+	/// just use a default one. The relay chain slot in the storage proof
+	/// will be adjusted to align with the parachain slot to pass validation.
+	///
+	/// Returns the block builder and validation data for further usage.
 	fn init_block_builder(
 		&self,
 		validation_data: Option<PersistedValidationData<PHash, PBlockNumber>>,
 		relay_sproof_builder: RelayStateSproofBuilder,
-	) -> sc_block_builder::BlockBuilder<Block, Client>;
+	) -> BlockBuilderAndSupportData;
 
 	/// Init a specific block builder at a specific block that works for the test runtime.
 	///
@@ -48,7 +63,7 @@ pub trait InitBlockBuilder {
 		at: Hash,
 		validation_data: Option<PersistedValidationData<PHash, PBlockNumber>>,
 		relay_sproof_builder: RelayStateSproofBuilder,
-	) -> sc_block_builder::BlockBuilder<Block, Client>;
+	) -> BlockBuilderAndSupportData;
 
 	/// Init a specific block builder that works for the test runtime.
 	///
@@ -61,21 +76,30 @@ pub trait InitBlockBuilder {
 		validation_data: Option<PersistedValidationData<PHash, PBlockNumber>>,
 		relay_sproof_builder: RelayStateSproofBuilder,
 		timestamp: u64,
-	) -> sc_block_builder::BlockBuilder<Block, Client>;
+	) -> BlockBuilderAndSupportData;
 }
 
 fn init_block_builder(
 	client: &Client,
 	at: Hash,
 	validation_data: Option<PersistedValidationData<PHash, PBlockNumber>>,
-	relay_sproof_builder: RelayStateSproofBuilder,
+	mut relay_sproof_builder: RelayStateSproofBuilder,
 	timestamp: u64,
-) -> BlockBuilder<'_, Block, Client> {
+) -> BlockBuilderAndSupportData<'_> {
+	// This slot will be used for both relay chain and parachain
+	let slot: Slot = (timestamp / cumulus_test_runtime::SLOT_DURATION).into();
+	relay_sproof_builder.current_slot = slot;
+
+	let aura_pre_digest = Digest {
+		logs: vec![DigestItem::PreRuntime(sp_consensus_aura::AURA_ENGINE_ID, slot.encode())],
+	};
+
 	let mut block_builder = BlockBuilderBuilder::new(client)
 		.on_parent_block(at)
 		.fetch_parent_block_number(client)
 		.unwrap()
 		.enable_proof_recording()
+		.with_inherent_digests(aura_pre_digest)
 		.build()
 		.expect("Creates new block builder for test runtime");
 
@@ -89,18 +113,13 @@ fn init_block_builder(
 		relay_sproof_builder.into_state_root_and_proof();
 
 	let mut validation_data = validation_data.unwrap_or_default();
-	assert_eq!(
-		validation_data.relay_parent_storage_root,
-		Default::default(),
-		"Overriding the relay storage root is not implemented",
-	);
 	validation_data.relay_parent_storage_root = relay_parent_storage_root;
 
 	inherent_data
 		.put_data(
 			INHERENT_IDENTIFIER,
 			&ParachainInherentData {
-				validation_data,
+				validation_data: validation_data.clone(),
 				relay_chain_state,
 				downward_messages: Default::default(),
 				horizontal_messages: Default::default(),
@@ -114,7 +133,7 @@ fn init_block_builder(
 		.into_iter()
 		.for_each(|ext| block_builder.push(ext).expect("Pushes inherent"));
 
-	block_builder
+	BlockBuilderAndSupportData { block_builder, persisted_validation_data: validation_data, slot }
 }
 
 impl InitBlockBuilder for Client {
@@ -122,7 +141,7 @@ impl InitBlockBuilder for Client {
 		&self,
 		validation_data: Option<PersistedValidationData<PHash, PBlockNumber>>,
 		relay_sproof_builder: RelayStateSproofBuilder,
-	) -> BlockBuilder<Block, Client> {
+	) -> BlockBuilderAndSupportData {
 		let chain_info = self.chain_info();
 		self.init_block_builder_at(chain_info.best_hash, validation_data, relay_sproof_builder)
 	}
@@ -132,10 +151,17 @@ impl InitBlockBuilder for Client {
 		at: Hash,
 		validation_data: Option<PersistedValidationData<PHash, PBlockNumber>>,
 		relay_sproof_builder: RelayStateSproofBuilder,
-	) -> BlockBuilder<Block, Client> {
+	) -> BlockBuilderAndSupportData {
 		let last_timestamp = self.runtime_api().get_last_timestamp(at).expect("Get last timestamp");
 
-		let timestamp = last_timestamp + cumulus_test_runtime::MinimumPeriod::get();
+		let timestamp = if last_timestamp == 0 {
+			std::time::SystemTime::now()
+				.duration_since(std::time::SystemTime::UNIX_EPOCH)
+				.expect("Time is always after UNIX_EPOCH; qed")
+				.as_millis() as u64
+		} else {
+			last_timestamp + cumulus_test_runtime::SLOT_DURATION
+		};
 
 		init_block_builder(self, at, validation_data, relay_sproof_builder, timestamp)
 	}
@@ -146,7 +172,7 @@ impl InitBlockBuilder for Client {
 		validation_data: Option<PersistedValidationData<PHash, PBlockNumber>>,
 		relay_sproof_builder: RelayStateSproofBuilder,
 		timestamp: u64,
-	) -> sc_block_builder::BlockBuilder<Block, Client> {
+	) -> BlockBuilderAndSupportData {
 		init_block_builder(self, at, validation_data, relay_sproof_builder, timestamp)
 	}
 }

cumulus/test/client/src/lib.rs

@@ -17,24 +17,30 @@
 //! A Cumulus test client.
 
 mod block_builder;
+pub use block_builder::*;
 use codec::{Decode, Encode};
+pub use cumulus_test_runtime as runtime;
+use cumulus_test_runtime::AuraId;
+pub use polkadot_parachain_primitives::primitives::{
+	BlockData, HeadData, ValidationParams, ValidationResult,
+};
 use runtime::{
 	Balance, Block, BlockHashCount, Runtime, RuntimeCall, Signature, SignedExtra, SignedPayload,
 	UncheckedExtrinsic, VERSION,
 };
+use sc_consensus_aura::standalone::{seal, slot_author};
+pub use sc_executor::error::Result as ExecutorResult;
 use sc_executor::HeapAllocStrategy;
 use sc_executor_common::runtime_blob::RuntimeBlob;
+use sp_api::ProvideRuntimeApi;
+use sp_application_crypto::AppCrypto;
 use sp_blockchain::HeaderBackend;
+use sp_consensus_aura::{AuraApi, Slot};
 use sp_core::Pair;
 use sp_io::TestExternalities;
-use sp_runtime::{generic::Era, BuildStorage, SaturatedConversion};
-
-pub use block_builder::*;
-pub use cumulus_test_runtime as runtime;
-pub use polkadot_parachain_primitives::primitives::{
-	BlockData, HeadData, ValidationParams, ValidationResult,
-};
-pub use sc_executor::error::Result as ExecutorResult;
+use sp_keystore::testing::MemoryKeystore;
+use sp_runtime::{generic::Era, traits::Header, BuildStorage, SaturatedConversion};
+use std::sync::Arc;
 pub use substrate_test_client::*;
 
 pub type ParachainBlockData = cumulus_primitives_core::ParachainBlockData<Block>;
@@ -225,3 +231,40 @@ pub fn validate_block(
 		)
 		.map(|v| ValidationResult::decode(&mut &v[..]).expect("Decode `ValidationResult`."))
 }
+
+fn get_keystore() -> sp_keystore::KeystorePtr {
+	let keystore = MemoryKeystore::new();
+	sp_keyring::Sr25519Keyring::iter().for_each(|key| {
+		keystore
+			.sr25519_generate_new(
+				sp_consensus_aura::sr25519::AuthorityPair::ID,
+				Some(&key.to_seed()),
+			)
+			.expect("Key should be created");
+	});
+	Arc::new(keystore)
+}
+
+/// Given parachain block data and a slot, seal the block with an aura seal. Assumes that the
+/// authorities of the test runtime are present in the keyring.
+pub fn seal_block(
+	block: ParachainBlockData,
+	parachain_slot: Slot,
+	client: &Client,
+) -> ParachainBlockData {
+	let parent_hash = block.header().parent_hash;
+	let authorities = client.runtime_api().authorities(parent_hash).unwrap();
+	let expected_author = slot_author::<<AuraId as AppCrypto>::Pair>(parachain_slot, &authorities)
+		.expect("Should be able to find author");
+
+	let (mut header, extrinsics, proof) = block.deconstruct();
+	let keystore = get_keystore();
+	let seal_digest = seal::<_, sp_consensus_aura::sr25519::AuthorityPair>(
+		&header.hash(),
+		expected_author,
+		&keystore,
+	)
+	.expect("Should be able to create seal");
+	header.digest_mut().push(seal_digest);
+	ParachainBlockData::new(header, extrinsics, proof)
+}