Remove requirement on Hash = H256, make Proposer return StorageChanges and Proof (#3860)

* Extend `Proposer` to optionally generate a proof of the proposal * Something * Refactor sr-api to not depend on client anymore * Fix benches * Apply suggestions from code review Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com> * Apply suggestions from code review * Introduce new `into_storage_changes` function * Switch to runtime api for `execute_block` and don't require `H256` anywhere in the code * Put the `StorageChanges` into the `Proposal` * Move the runtime api error to its own trait * Adds `StorageTransactionCache` to the runtime api This requires that we add `type NodeBlock = ` to the `impl_runtime_apis!` macro to work around some bugs in rustc :( * Remove `type NodeBlock` and switch to a "better" hack * Start using the transaction cache from the runtime api * Make it compile * Move `InMemory` to its own file * Make all tests work again * Return block, storage_changes and proof from Blockbuilder::bake() * Make sure that we use/set `storage_changes` when possible * Add test * Fix deadlock * Remove accidentally added folders * Introduce `RecordProof` as argument type to be more explicit * Update client/src/client.rs Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com> * Update primitives/state-machine/src/ext.rs Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com> * Integrates review feedback * Remove `unsafe` usage * Update client/block-builder/src/lib.rs Co-Authored-By: Benjamin Kampmann <ben@gnunicorn.org> * Update client/src/call_executor.rs * Bump versions Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com> Co-authored-by: Benjamin Kampmann <ben.kampmann@googlemail.com>
2026-04-28 14:27:57 +00:00 · 2020-01-10 10:48:32 +01:00
parent 74d6e660c6
commit fd6b29dd2c
140 changed files with 4860 additions and 3339 deletions
@@ -21,9 +21,7 @@ mod block_import;
 #[cfg(test)]
 mod sync;

-use std::collections::HashMap;
-use std::pin::Pin;
-use std::sync::Arc;
+use std::{collections::HashMap, pin::Pin, sync::Arc, marker::PhantomData};

 use libp2p::build_multiaddr;
 use log::trace;
@@ -35,15 +33,14 @@ use sc_client_api::{
 	BlockchainEvents, BlockImportNotification,
 	FinalityNotifications, ImportNotifications,
 	FinalityNotification,
-	backend::{AuxStore, Backend, Finalizer}
+	backend::{TransactionFor, AuxStore, Backend, Finalizer},
 };
 use sc_block_builder::BlockBuilder;
 use sc_client::LongestChain;
 use sc_network::config::Roles;
 use sp_consensus::block_validation::DefaultBlockAnnounceValidator;
-use sp_consensus::import_queue::BasicQueue;
 use sp_consensus::import_queue::{
-	BoxBlockImport, BoxJustificationImport, Verifier, BoxFinalityProofImport,
+	BasicQueue, BoxJustificationImport, Verifier, BoxFinalityProofImport,
 };
 use sp_consensus::block_import::{BlockImport, ImportResult};
 use sp_consensus::Error as ConsensusError;
@@ -57,7 +54,7 @@ use parking_lot::Mutex;
 use sp_core::H256;
 use sc_network::{Context, ProtocolConfig};
 use sp_runtime::generic::{BlockId, OpaqueDigestItemId};
-use sp_runtime::traits::{Block as BlockT, Header, NumberFor};
+use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor};
 use sp_runtime::Justification;
 use sc_network::TransactionPool;
 use sc_network::specialization::NetworkSpecialization;
@@ -81,7 +78,7 @@ impl<B: BlockT> Verifier<B> for PassThroughVerifier {
 		header: B::Header,
 		justification: Option<Justification>,
 		body: Option<Vec<B::Extrinsic>>
-	) -> Result<(BlockImportParams<B>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String> {
+	) -> Result<(BlockImportParams<B, ()>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String> {
 		let maybe_keys = header.digest()
 			.log(|l| l.try_as_raw(OpaqueDigestItemId::Consensus(b"aura"))
 				.or_else(|| l.try_as_raw(OpaqueDigestItemId::Consensus(b"babe")))
@@ -92,6 +89,7 @@ impl<B: BlockT> Verifier<B> for PassThroughVerifier {
 			origin,
 			header,
 			body,
+			storage_changes: None,
 			finalized: self.0,
 			justification,
 			post_digests: vec![],
@@ -148,10 +146,12 @@ impl PeersClient {
 		}
 	}

-	pub fn as_block_import(&self) -> BoxBlockImport<Block> {
+	pub fn as_block_import<Transaction>(&self) -> BlockImportAdapter<Transaction> {
 		match *self {
-			PeersClient::Full(ref client, ref _backend) => Box::new(client.clone()) as _,
-			PeersClient::Light(ref client, ref _backend) => Box::new(client.clone()) as _,
+			PeersClient::Full(ref client, ref _backend) =>
+				BlockImportAdapter::new_full(client.clone()),
+			PeersClient::Light(ref client, ref _backend) =>
+				BlockImportAdapter::Light(Arc::new(Mutex::new(client.clone())), PhantomData),
 		}
 	}

@@ -218,7 +218,7 @@ pub struct Peer<D, S: NetworkSpecialization<Block>> {
 	verifier: VerifierAdapter<dyn Verifier<Block>>,
 	/// We keep a copy of the block_import so that we can invoke it for locally-generated blocks,
 	/// instead of going through the import queue.
-	block_import: Box<dyn BlockImport<Block, Error = ConsensusError>>,
+	block_import: BlockImportAdapter<()>,
 	select_chain: Option<LongestChain<substrate_test_runtime_client::Backend, Block>>,
 	backend: Option<Arc<substrate_test_runtime_client::Backend>>,
 	network: NetworkWorker<Block, S, <Block as BlockT>::Hash>,
@@ -269,7 +269,7 @@ impl<D, S: NetworkSpecialization<Block>> Peer<D, S> {

 	/// Add blocks to the peer -- edit the block before adding
 	pub fn generate_blocks<F>(&mut self, count: usize, origin: BlockOrigin, edit_block: F) -> H256
-		where F: FnMut(BlockBuilder<Block, PeersFullClient>) -> Block
+		where F: FnMut(BlockBuilder<Block, PeersFullClient, substrate_test_runtime_client::Backend>) -> Block
 	{
 		let best_hash = self.client.info().best_hash;
 		self.generate_blocks_at(BlockId::Hash(best_hash), count, origin, edit_block)
@@ -283,11 +283,15 @@ impl<D, S: NetworkSpecialization<Block>> Peer<D, S> {
 		count: usize,
 		origin: BlockOrigin,
 		mut edit_block: F
-	) -> H256 where F: FnMut(BlockBuilder<Block, PeersFullClient>) -> Block {
-		let full_client = self.client.as_full().expect("blocks could only be generated by full clients");
+	) -> H256 where F: FnMut(BlockBuilder<Block, PeersFullClient, substrate_test_runtime_client::Backend>) -> Block {
+		let full_client = self.client.as_full()
+			.expect("blocks could only be generated by full clients");
 		let mut at = full_client.header(&at).unwrap().unwrap().hash();
 		for _  in 0..count {
-			let builder = full_client.new_block_at(&BlockId::Hash(at), Default::default()
+			let builder = full_client.new_block_at(
+				&BlockId::Hash(at),
+				Default::default(),
+				false,
 			).unwrap();
 			let block = edit_block(builder);
 			let hash = block.header.hash();
@@ -296,7 +300,7 @@ impl<D, S: NetworkSpecialization<Block>> Peer<D, S> {
 				"Generating {}, (#{}, parent={})",
 				hash,
 				block.header.number,
-				block.header.parent_hash
+				block.header.parent_hash,
 			);
 			let header = block.header.clone();
 			let (import_block, cache) = self.verifier.verify(
@@ -339,17 +343,22 @@ impl<D, S: NetworkSpecialization<Block>> Peer<D, S> {
 				};
 				builder.push(transfer.into_signed_tx()).unwrap();
 				nonce = nonce + 1;
-				builder.bake().unwrap()
+				builder.build().unwrap().block
 			})
 		} else {
-			self.generate_blocks_at(at, count, BlockOrigin::File, |builder| builder.bake().unwrap())
+			self.generate_blocks_at(
+				at,
+				count,
+				BlockOrigin::File,
+				|builder| builder.build().unwrap().block,
+			)
 		}
 	}

 	pub fn push_authorities_change_block(&mut self, new_authorities: Vec<AuthorityId>) -> H256 {
 		self.generate_blocks(1, BlockOrigin::File, |mut builder| {
 			builder.push(Extrinsic::AuthoritiesChange(new_authorities.clone())).unwrap();
-			builder.bake().unwrap()
+			builder.build().unwrap().block
 		})
 	}

@@ -418,33 +427,90 @@ impl SpecializationFactory for DummySpecialization {
 	}
 }

-/// Implements `BlockImport` on an `Arc<Mutex<impl BlockImport>>`. Used internally. Necessary to overcome the way the
-/// `TestNet` trait is designed, more specifically `make_block_import` returning a `Box<BlockImport>` makes it
-/// impossible to clone the underlying object.
-struct BlockImportAdapter<T: ?Sized>(Arc<Mutex<Box<T>>>);
+/// Implements `BlockImport` for any `Transaction`. Internally the transaction is
+/// "converted", aka the field is set to `None`.
+///
+/// This is required as the `TestNetFactory` trait does not distinguish between
+/// full and light nodes.
+pub enum BlockImportAdapter<Transaction> {
+	Full(
+		Arc<Mutex<dyn BlockImport<
+			Block,
+			Transaction = TransactionFor<substrate_test_runtime_client::Backend, Block>,
+			Error = ConsensusError
+		> + Send>>,
+		PhantomData<Transaction>,
+	),
+	Light(
+		Arc<Mutex<dyn BlockImport<
+			Block,
+			Transaction = TransactionFor<substrate_test_runtime_client::LightBackend, Block>,
+			Error = ConsensusError
+		> + Send>>,
+		PhantomData<Transaction>,
+	),
+}

-impl<T: ?Sized> Clone for BlockImportAdapter<T> {
-	fn clone(&self) -> Self {
-		BlockImportAdapter(self.0.clone())
+impl<Transaction> BlockImportAdapter<Transaction> {
+	/// Create a new instance of `Self::Full`.
+	pub fn new_full(
+		full: impl BlockImport<
+			Block,
+			Transaction = TransactionFor<substrate_test_runtime_client::Backend, Block>,
+			Error = ConsensusError
+		>
+		+ 'static
+		+ Send
+	) -> Self {
+		Self::Full(Arc::new(Mutex::new(full)), PhantomData)
+	}
+
+	/// Create a new instance of `Self::Light`.
+	pub fn new_light(
+		light: impl BlockImport<
+			Block,
+			Transaction = TransactionFor<substrate_test_runtime_client::LightBackend, Block>,
+			Error = ConsensusError
+		>
+		+ 'static
+		+ Send
+	) -> Self {
+		Self::Light(Arc::new(Mutex::new(light)), PhantomData)
 	}
 }

-impl<T: ?Sized + BlockImport<Block>> BlockImport<Block> for BlockImportAdapter<T> {
-	type Error = T::Error;
+impl<Transaction> Clone for BlockImportAdapter<Transaction> {
+	fn clone(&self) -> Self {
+		match self {
+			Self::Full(full, _) => Self::Full(full.clone(), PhantomData),
+			Self::Light(light, _) => Self::Light(light.clone(), PhantomData),
+		}
+	}
+}
+
+impl<Transaction> BlockImport<Block> for BlockImportAdapter<Transaction> {
+	type Error = ConsensusError;
+	type Transaction = Transaction;

 	fn check_block(
 		&mut self,
 		block: BlockCheckParams<Block>,
 	) -> Result<ImportResult, Self::Error> {
-		self.0.lock().check_block(block)
+		match self {
+			Self::Full(full, _) => full.lock().check_block(block),
+			Self::Light(light, _) => light.lock().check_block(block),
+		}
 	}

 	fn import_block(
 		&mut self,
-		block: BlockImportParams<Block>,
+		block: BlockImportParams<Block, Transaction>,
 		cache: HashMap<well_known_cache_keys::Id, Vec<u8>>,
 	) -> Result<ImportResult, Self::Error> {
-		self.0.lock().import_block(block, cache)
+		match self {
+			Self::Full(full, _) => full.lock().import_block(block.convert_transaction(), cache),
+			Self::Light(light, _) => light.lock().import_block(block.convert_transaction(), cache),
+		}
 	}
 }

@@ -464,7 +530,7 @@ impl<B: BlockT, T: ?Sized + Verifier<B>> Verifier<B> for VerifierAdapter<T> {
 		header: B::Header,
 		justification: Option<Justification>,
 		body: Option<Vec<B::Extrinsic>>
-	) -> Result<(BlockImportParams<B>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String> {
+	) -> Result<(BlockImportParams<B, ()>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String> {
 		self.0.lock().verify(origin, header, justification, body)
 	}
 }
@@ -486,12 +552,15 @@ pub trait TestNetFactory: Sized {
 	/// Get reference to peer.
 	fn peer(&mut self, i: usize) -> &mut Peer<Self::PeerData, Self::Specialization>;
 	fn peers(&self) -> &Vec<Peer<Self::PeerData, Self::Specialization>>;
-	fn mut_peers<F: FnOnce(&mut Vec<Peer<Self::PeerData, Self::Specialization>>)>(&mut self, closure: F);
+	fn mut_peers<F: FnOnce(&mut Vec<Peer<Self::PeerData, Self::Specialization>>)>(
+		&mut self,
+		closure: F,
+	);

 	/// Get custom block import handle for fresh client, along with peer data.
-	fn make_block_import(&self, client: PeersClient)
+	fn make_block_import<Transaction>(&self, client: PeersClient)
 		-> (
-			BoxBlockImport<Block>,
+			BlockImportAdapter<Transaction>,
 			Option<BoxJustificationImport<Block>>,
 			Option<BoxFinalityProofImport<Block>>,
 			Option<BoxFinalityProofRequestBuilder<Block>>,
@@ -502,7 +571,10 @@ pub trait TestNetFactory: Sized {
 	}

 	/// Get finality proof provider (if supported).
-	fn make_finality_proof_provider(&self, _client: PeersClient) -> Option<Arc<dyn FinalityProofProvider<Block>>> {
+	fn make_finality_proof_provider(
+		&self,
+		_client: PeersClient,
+	) -> Option<Arc<dyn FinalityProofProvider<Block>>> {
 		None
 	}

@@ -544,7 +616,6 @@ pub trait TestNetFactory: Sized {
 			finality_proof_request_builder,
 			data,
 		) = self.make_block_import(PeersClient::Full(client.clone(), backend.clone()));
-		let block_import = BlockImportAdapter(Arc::new(Mutex::new(block_import)));

 		let verifier = self.make_verifier(
 			PeersClient::Full(client.clone(), backend.clone()),
@@ -570,7 +641,9 @@ pub trait TestNetFactory: Sized {
 				..NetworkConfiguration::default()
 			},
 			chain: client.clone(),
-			finality_proof_provider: self.make_finality_proof_provider(PeersClient::Full(client.clone(), backend.clone())),
+			finality_proof_provider: self.make_finality_proof_provider(
+				PeersClient::Full(client.clone(), backend.clone()),
+			),
 			finality_proof_request_builder,
 			on_demand: None,
 			transaction_pool: Arc::new(EmptyTransactionPool),
@@ -597,7 +670,7 @@ pub trait TestNetFactory: Sized {
 				backend: Some(backend),
 				imported_blocks_stream,
 				finality_notification_stream,
-				block_import: Box::new(block_import),
+				block_import,
 				verifier,
 				network,
 			});
@@ -618,7 +691,6 @@ pub trait TestNetFactory: Sized {
 			finality_proof_request_builder,
 			data,
 		) = self.make_block_import(PeersClient::Light(client.clone(), backend.clone()));
-		let block_import = BlockImportAdapter(Arc::new(Mutex::new(block_import)));

 		let verifier = self.make_verifier(
 			PeersClient::Light(client.clone(), backend.clone()),
@@ -644,7 +716,9 @@ pub trait TestNetFactory: Sized {
 				..NetworkConfiguration::default()
 			},
 			chain: client.clone(),
-			finality_proof_provider: self.make_finality_proof_provider(PeersClient::Light(client.clone(), backend.clone())),
+			finality_proof_provider: self.make_finality_proof_provider(
+				PeersClient::Light(client.clone(), backend.clone())
+			),
 			finality_proof_request_builder,
 			on_demand: None,
 			transaction_pool: Arc::new(EmptyTransactionPool),
@@ -669,7 +743,7 @@ pub trait TestNetFactory: Sized {
 				verifier,
 				select_chain: None,
 				backend: None,
-				block_import: Box::new(block_import),
+				block_import,
 				client: PeersClient::Light(client, backend),
 				imported_blocks_stream,
 				finality_notification_stream,
@@ -721,7 +795,12 @@ pub trait TestNetFactory: Sized {

 				// We poll `imported_blocks_stream`.
 				while let Ok(Async::Ready(Some(notification))) = peer.imported_blocks_stream.poll() {
-					peer.network.on_block_imported(notification.hash, notification.header, Vec::new(), true);
+					peer.network.on_block_imported(
+						notification.hash,
+						notification.header,
+						Vec::new(),
+						true,
+					);
 				}

 				// We poll `finality_notification_stream`, but we only take the last event.
@@ -811,19 +890,28 @@ impl TestNetFactory for JustificationTestNet {
 		self.0.peers()
 	}

-	fn mut_peers<F: FnOnce(&mut Vec<Peer<Self::PeerData, Self::Specialization>>)>(&mut self, closure: F) {
+	fn mut_peers<F: FnOnce(
+		&mut Vec<Peer<Self::PeerData,
+		Self::Specialization>>,
+	)>(&mut self, closure: F) {
 		self.0.mut_peers(closure)
 	}

-	fn make_block_import(&self, client: PeersClient)
+	fn make_block_import<Transaction>(&self, client: PeersClient)
 		-> (
-			BoxBlockImport<Block>,
+			BlockImportAdapter<Transaction>,
 			Option<BoxJustificationImport<Block>>,
 			Option<BoxFinalityProofImport<Block>>,
 			Option<BoxFinalityProofRequestBuilder<Block>>,
 			Self::PeerData,
 		)
 	{
-		(client.as_block_import(), Some(Box::new(ForceFinalized(client))), None, None, Default::default())
+		(
+			client.as_block_import(),
+			Some(Box::new(ForceFinalized(client))),
+			None,
+			None,
+			Default::default(),
+		)
 	}
 }