Update Cumulus for Parachains V1 (#224)

* Start with something * Whatever * Update * MOARE * Make cumulus-network compile and tests work * Update more and fixes * More stuff * More fixes * Make collator build * Make test almost work * Remove contracts runtime * More test work * Make service compile * Fix test-service * Fix test client * More fixes * Fix collator test * Fix network tests (again) * Make everything compile, finally * Fix tests * Update to latest masters * Remove ignore * Switch to different branch in polkadot for now * Update reference * Make it compile with latest changes * Update collator/src/lib.rs Co-authored-by: Robert Habermeier <rphmeier@gmail.com> * Update to latest upstream * Update to latest master * Fix test Co-authored-by: Robert Habermeier <rphmeier@gmail.com>
2026-05-30 14:01:02 +00:00 · 2020-11-08 22:18:09 +01:00
parent 55364ac254
commit 28338431fe
87 changed files with 3815 additions and 17343 deletions
@@ -9,19 +9,21 @@ edition = "2018"
 cumulus-consensus = { path = "../consensus" }
 cumulus-collator = { path = "../collator" }
 cumulus-primitives = { path = "../primitives" }
-cumulus-network = { path = "../network" }

 # Substrate dependencies
-sc-service = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
-sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
-sp-consensus = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
-sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
-sp-api = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
-sp-core = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
-sp-inherents = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
-sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "rococo-branch" }
+sc-service = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-consensus = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-inherents = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" }

 # Polkadot dependencies
-polkadot-collator = { git = "https://github.com/paritytech/polkadot", branch = "rococo-branch" }
-polkadot-primitives = { git = "https://github.com/paritytech/polkadot", branch = "rococo-branch" }
-polkadot-service = { git = "https://github.com/paritytech/polkadot", branch = "rococo-branch" }
+polkadot-primitives = { git = "https://github.com/paritytech/polkadot", branch = "bkchr-adder-collator-integration-test" }
+polkadot-service = { git = "https://github.com/paritytech/polkadot", branch = "bkchr-adder-collator-integration-test" }
+polkadot-overseer = { git = "https://github.com/paritytech/polkadot", branch = "bkchr-adder-collator-integration-test" }
+
+# Other deps
+futures = "0.3.6"
@@ -18,33 +18,37 @@
 //!
 //! Provides functions for starting a collator node or a normal full node.

-use cumulus_collator::CollatorBuilder;
-use cumulus_network::{DelayedBlockAnnounceValidator, JustifiedBlockAnnounceValidator};
 use cumulus_primitives::ParaId;
-use polkadot_primitives::v0::{Block as PBlock, CollatorPair};
-use polkadot_service::{AbstractClient, ClientHandle, RuntimeApiCollection};
+use futures::{Future, FutureExt};
+use polkadot_overseer::OverseerHandler;
+use polkadot_primitives::v1::{Block as PBlock, CollatorId, CollatorPair};
+use polkadot_service::{AbstractClient, Client as PClient, ClientHandle, RuntimeApiCollection};
 use sc_client_api::{Backend as BackendT, BlockBackend, Finalizer, UsageProvider};
-use sc_service::{Configuration, Role, TaskManager};
-use sp_blockchain::{HeaderBackend, Result as ClientResult};
-use sp_consensus::{BlockImport, Environment, Error as ConsensusError, Proposer, SyncOracle};
-use sp_core::crypto::Pair;
+use sc_service::{error::Result as ServiceResult, Configuration, Role, TaskManager};
+use sp_blockchain::HeaderBackend;
+use sp_consensus::{BlockImport, Environment, Error as ConsensusError, Proposer};
+use sp_core::traits::SpawnNamed;
 use sp_inherents::InherentDataProviders;
 use sp_runtime::traits::{BlakeTwo256, Block as BlockT};
 use std::{marker::PhantomData, sync::Arc};

+/// Polkadot full node handles.
+type PFullNode<C> = polkadot_service::NewFull<C>;
+
 /// Parameters given to [`start_collator`].
-pub struct StartCollatorParams<'a, Block: BlockT, PF, BI, BS, Client> {
-	pub para_id: ParaId,
+pub struct StartCollatorParams<'a, Block: BlockT, PF, BI, BS, Client, Backend, Spawner, PClient> {
 	pub proposer_factory: PF,
 	pub inherent_data_providers: InherentDataProviders,
+	pub backend: Arc<Backend>,
 	pub block_import: BI,
 	pub block_status: Arc<BS>,
-	pub announce_block: Arc<dyn Fn(Block::Hash, Vec<u8>) + Send + Sync>,
 	pub client: Arc<Client>,
-	pub block_announce_validator: DelayedBlockAnnounceValidator<Block>,
+	pub announce_block: Arc<dyn Fn(Block::Hash, Vec<u8>) + Send + Sync>,
+	pub spawner: Spawner,
+	pub para_id: ParaId,
+	pub collator_key: CollatorPair,
+	pub polkadot_full_node: PFullNode<PClient>,
 	pub task_manager: &'a mut TaskManager,
-	pub polkadot_config: Configuration,
-	pub collator_key: Arc<CollatorPair>,
 }

 /// Start a collator node for a parachain.
@@ -52,20 +56,21 @@ pub struct StartCollatorParams<'a, Block: BlockT, PF, BI, BS, Client> {
 /// A collator is similar to a validator in a normal blockchain.
 /// It is responsible for producing blocks and sending the blocks to a
 /// parachain validator for validation and inclusion into the relay chain.
-pub fn start_collator<'a, Block, PF, BI, BS, Client, Backend>(
+pub async fn start_collator<'a, Block, PF, BI, BS, Client, Backend, Spawner, PClient>(
 	StartCollatorParams {
-		para_id,
 		proposer_factory,
 		inherent_data_providers,
+		backend,
 		block_import,
 		block_status,
-		announce_block,
 		client,
-		block_announce_validator,
-		task_manager,
-		polkadot_config,
+		announce_block,
+		spawner,
+		para_id,
 		collator_key,
-	}: StartCollatorParams<'a, Block, PF, BI, BS, Client>,
+		polkadot_full_node,
+		task_manager,
+	}: StartCollatorParams<'a, Block, PF, BI, BS, Client, Backend, Spawner, PClient>,
 ) -> sc_service::error::Result<()>
 where
 	Block: BlockT,
@@ -87,55 +92,124 @@ where
 		+ 'static,
 	for<'b> &'b Client: BlockImport<Block>,
 	Backend: BackendT<Block> + 'static,
+	Spawner: SpawnNamed + Clone + Send + Sync + 'static,
+	PClient: ClientHandle,
 {
-	let builder = CollatorBuilder::new(
-		proposer_factory,
-		inherent_data_providers,
-		block_import,
-		block_status,
-		para_id,
-		client,
-		announce_block,
-		block_announce_validator,
-	);
+	polkadot_full_node
+		.client
+		.execute_with(StartCollator {
+			proposer_factory,
+			inherent_data_providers,
+			backend,
+			client,
+			announce_block,
+			overseer_handler: polkadot_full_node
+				.overseer_handler
+				.ok_or_else(|| "Polkadot full node did not provided an `OverseerHandler`!")?,
+			spawner,
+			para_id,
+			collator_key,
+			block_import,
+			block_status,
+		})
+		.await?;

-	let (polkadot_future, polkadot_task_manager) =
-		polkadot_collator::start_collator(builder, para_id, collator_key, polkadot_config)?;
-
-	task_manager
-		.spawn_essential_handle()
-		.spawn("polkadot", polkadot_future);
-
-	task_manager.add_child(polkadot_task_manager);
+	task_manager.add_child(polkadot_full_node.task_manager);

 	Ok(())
 }

+struct StartCollator<Block: BlockT, Client, Backend, PF, BI, BS, Spawner> {
+	proposer_factory: PF,
+	inherent_data_providers: InherentDataProviders,
+	backend: Arc<Backend>,
+	block_import: BI,
+	block_status: Arc<BS>,
+	client: Arc<Client>,
+	announce_block: Arc<dyn Fn(Block::Hash, Vec<u8>) + Send + Sync>,
+	overseer_handler: OverseerHandler,
+	spawner: Spawner,
+	para_id: ParaId,
+	collator_key: CollatorPair,
+}
+
+impl<Block, Client, Backend, PF, BI, BS, Spawner> polkadot_service::ExecuteWithClient
+	for StartCollator<Block, Client, Backend, PF, BI, BS, Spawner>
+where
+	Block: BlockT,
+	PF: Environment<Block> + Send + 'static,
+	BI: BlockImport<
+			Block,
+			Error = ConsensusError,
+			Transaction = <PF::Proposer as Proposer<Block>>::Transaction,
+		> + Send
+		+ Sync
+		+ 'static,
+	BS: BlockBackend<Block> + Send + Sync + 'static,
+	Client: Finalizer<Block, Backend>
+		+ UsageProvider<Block>
+		+ HeaderBackend<Block>
+		+ Send
+		+ Sync
+		+ BlockBackend<Block>
+		+ 'static,
+	for<'b> &'b Client: BlockImport<Block>,
+	Backend: BackendT<Block> + 'static,
+	Spawner: SpawnNamed + Clone + Send + Sync + 'static,
+{
+	type Output = std::pin::Pin<Box<dyn Future<Output = ServiceResult<()>>>>;
+
+	fn execute_with_client<PClient, Api, PBackend>(self, client: Arc<PClient>) -> Self::Output
+	where
+		<Api as sp_api::ApiExt<PBlock>>::StateBackend: sp_api::StateBackend<BlakeTwo256>,
+		PBackend: sc_client_api::Backend<PBlock>,
+		PBackend::State: sp_api::StateBackend<BlakeTwo256>,
+		Api: RuntimeApiCollection<StateBackend = PBackend::State>,
+		PClient: AbstractClient<PBlock, PBackend, Api = Api> + 'static,
+	{
+		async move {
+			cumulus_collator::start_collator(cumulus_collator::StartCollatorParams {
+				proposer_factory: self.proposer_factory,
+				inherent_data_providers: self.inherent_data_providers,
+				backend: self.backend,
+				block_import: self.block_import,
+				block_status: self.block_status,
+				client: self.client,
+				announce_block: self.announce_block,
+				overseer_handler: self.overseer_handler,
+				spawner: self.spawner,
+				para_id: self.para_id,
+				key: self.collator_key,
+				polkadot_client: client,
+			})
+			.await
+			.map_err(Into::into)
+		}
+		.boxed()
+	}
+}
+
 /// Parameters given to [`start_full_node`].
-pub struct StartFullNodeParams<'a, Block: BlockT, Client> {
-	pub polkadot_config: Configuration,
-	pub collator_key: Arc<CollatorPair>,
+pub struct StartFullNodeParams<'a, Block: BlockT, Client, PClient> {
 	pub para_id: ParaId,
-	pub block_announce_validator: DelayedBlockAnnounceValidator<Block>,
 	pub client: Arc<Client>,
-	pub announce_block: Arc<dyn Fn(Block::Hash, Vec<u8>) + Send + Sync>,
+	pub polkadot_full_node: PFullNode<PClient>,
 	pub task_manager: &'a mut TaskManager,
+	pub announce_block: Arc<dyn Fn(Block::Hash, Vec<u8>) + Send + Sync>,
 }

 /// Start a full node for a parachain.
 ///
 /// A full node will only sync the given parachain and will follow the
 /// tip of the chain.
-pub fn start_full_node<Block, Client, Backend>(
+pub fn start_full_node<Block, Client, Backend, PClient>(
 	StartFullNodeParams {
-		polkadot_config,
-		collator_key,
-		para_id,
-		block_announce_validator,
 		client,
 		announce_block,
 		task_manager,
-	}: StartFullNodeParams<Block, Client>,
+		polkadot_full_node,
+		para_id,
+	}: StartFullNodeParams<Block, Client, PClient>,
 ) -> sc_service::error::Result<()>
 where
 	Block: BlockT,
@@ -147,54 +221,23 @@ where
 		+ 'static,
 	for<'a> &'a Client: BlockImport<Block>,
 	Backend: BackendT<Block> + 'static,
+	PClient: ClientHandle,
 {
-	let is_light = matches!(polkadot_config.role, Role::Light);
-	let (polkadot_task_manager, pclient, handles) = if is_light {
-		Err("Light client not supported.".into())
-	} else {
-		polkadot_service::build_full(
-			polkadot_config,
-			Some((collator_key.public(), para_id)),
-			None,
-			false,
-			6000,
-			None,
-		)
-	}?;
-
-	let polkadot_network = handles
-		.polkadot_network
-		.expect("Polkadot service is started; qed");
-
-	pclient.execute_with(InitParachainFullNode {
-		block_announce_validator,
-		para_id,
-		polkadot_sync_oracle: Box::new(polkadot_network),
+	polkadot_full_node.client.execute_with(StartFullNode {
 		announce_block,
+		para_id,
 		client,
 		task_manager,
 		_phantom: PhantomData,
 	})?;

-	task_manager.add_child(polkadot_task_manager);
+	task_manager.add_child(polkadot_full_node.task_manager);

 	Ok(())
 }

-/// Prepare the parachain's node condifugration
-///
-/// This function will disable the default announcement of Substrate for the parachain in favor
-/// of the one of Cumulus.
-pub fn prepare_node_config(mut parachain_config: Configuration) -> Configuration {
-	parachain_config.announce_block = false;
-
-	parachain_config
-}
-
-struct InitParachainFullNode<'a, Block: BlockT, Client, Backend> {
-	block_announce_validator: DelayedBlockAnnounceValidator<Block>,
+struct StartFullNode<'a, Block: BlockT, Client, Backend> {
 	para_id: ParaId,
-	polkadot_sync_oracle: Box<dyn SyncOracle + Send>,
 	announce_block: Arc<dyn Fn(Block::Hash, Vec<u8>) + Send + Sync>,
 	client: Arc<Client>,
 	task_manager: &'a mut TaskManager,
@@ -202,7 +245,7 @@ struct InitParachainFullNode<'a, Block: BlockT, Client, Backend> {
 }

 impl<'a, Block, Client, Backend> polkadot_service::ExecuteWithClient
-	for InitParachainFullNode<'a, Block, Client, Backend>
+	for StartFullNode<'a, Block, Client, Backend>
 where
 	Block: BlockT,
 	Client: Finalizer<Block, Backend>
@@ -214,7 +257,7 @@ where
 	for<'b> &'b Client: BlockImport<Block>,
 	Backend: BackendT<Block> + 'static,
 {
-	type Output = ClientResult<()>;
+	type Output = ServiceResult<()>;

 	fn execute_with_client<PClient, Api, PBackend>(self, client: Arc<PClient>) -> Self::Output
 	where
@@ -224,13 +267,6 @@ where
 		Api: RuntimeApiCollection<StateBackend = PBackend::State>,
 		PClient: AbstractClient<PBlock, PBackend, Api = Api> + 'static,
 	{
-		self.block_announce_validator
-			.set(Box::new(JustifiedBlockAnnounceValidator::new(
-				client.clone(),
-				self.para_id,
-				self.polkadot_sync_oracle,
-			)));
-
 		let future = cumulus_consensus::follow_polkadot(
 			self.para_id,
 			self.client,
@@ -244,3 +280,31 @@ where
 		Ok(())
 	}
 }
+
+/// Prepare the parachain's node condifugration
+///
+/// This function will disable the default announcement of Substrate for the parachain in favor
+/// of the one of Cumulus.
+pub fn prepare_node_config(mut parachain_config: Configuration) -> Configuration {
+	parachain_config.announce_block = false;
+
+	parachain_config
+}
+
+/// Build the Polkadot full node using the given `config`.
+pub fn build_polkadot_full_node(
+	config: Configuration,
+	collator_id: CollatorId,
+) -> sc_service::error::Result<PFullNode<PClient>> {
+	let is_light = matches!(config.role, Role::Light);
+	if is_light {
+		Err("Light client not supported.".into())
+	} else {
+		polkadot_service::build_full(
+			config,
+			polkadot_service::IsCollator::Yes(collator_id),
+			None,
+			None,
+		)
+	}
+}