Reorganising the repository - external renames and moves (#4074)

* Adding first rough ouline of the repository structure * Remove old CI stuff * add title * formatting fixes * move node-exits job's script to scripts dir * Move docs into subdir * move to bin * move maintainence scripts, configs and helpers into its own dir * add .local to ignore * move core->client * start up 'test' area * move test client * move test runtime * make test move compile * Add dependencies rule enforcement. * Fix indexing. * Update docs to reflect latest changes * Moving /srml->/paint * update docs * move client/sr-* -> primitives/ * clean old readme * remove old broken code in rhd * update lock * Step 1. * starting to untangle client * Fix after merge. * start splitting out client interfaces * move children and blockchain interfaces * Move trie and state-machine to primitives. * Fix WASM builds. * fixing broken imports * more interface moves * move backend and light to interfaces * move CallExecutor * move cli off client * moving around more interfaces * re-add consensus crates into the mix * fix subkey path * relieve client from executor * starting to pull out client from grandpa * move is_decendent_of out of client * grandpa still depends on client directly * lemme tests pass * rename srml->paint * Make it compile. * rename interfaces->client-api * Move keyring to primitives. * fixup libp2p dep * fix broken use * allow dependency enforcement to fail * move fork-tree * Moving wasm-builder * make env * move build-script-utils * fixup broken crate depdencies and names * fix imports for authority discovery * fix typo * update cargo.lock * fixing imports * Fix paths and add missing crates * re-add missing crates
2026-04-27 04:37:57 +00:00 · 2019-11-14 21:51:17 +01:00
parent becc3b0a4f
commit 60e5011c72
809 changed files with 7801 additions and 6464 deletions
@@ -0,0 +1,408 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{mem, pin::Pin, time::Duration};
+use futures::{prelude::*, channel::mpsc, task::Context, task::Poll};
+use futures_timer::Delay;
+use sr_primitives::{Justification, traits::{Block as BlockT, Header as HeaderT, NumberFor}};
+
+use crate::block_import::BlockOrigin;
+use crate::import_queue::{
+	BlockImportResult, BlockImportError, Verifier, BoxBlockImport, BoxFinalityProofImport,
+	BoxJustificationImport, ImportQueue, Link, Origin,
+	IncomingBlock, import_single_block,
+	buffered_link::{self, BufferedLinkSender, BufferedLinkReceiver}
+};
+
+/// Interface to a basic block import queue that is importing blocks sequentially in a separate
+/// task, with pluggable verification.
+pub struct BasicQueue<B: BlockT> {
+	/// Channel to send messages to the background task.
+	sender: mpsc::UnboundedSender<ToWorkerMsg<B>>,
+	/// Results coming from the worker task.
+	result_port: BufferedLinkReceiver<B>,
+	/// If it isn't possible to spawn the future in `future_to_spawn` (which is notably the case in
+	/// "no std" environment), we instead put it in `manual_poll`. It is then polled manually from
+	/// `poll_actions`.
+	manual_poll: Option<Pin<Box<dyn Future<Output = ()> + Send>>>,
+	/// A thread pool where the background worker is being run.
+	pool: Option<futures::executor::ThreadPool>,
+}
+
+impl<B: BlockT> BasicQueue<B> {
+	/// Instantiate a new basic queue, with given verifier.
+	///
+	/// This creates a background task, and calls `on_start` on the justification importer and
+	/// finality proof importer.
+	pub fn new<V: 'static + Verifier<B>>(
+		verifier: V,
+		block_import: BoxBlockImport<B>,
+		justification_import: Option<BoxJustificationImport<B>>,
+		finality_proof_import: Option<BoxFinalityProofImport<B>>,
+	) -> Self {
+		let (result_sender, result_port) = buffered_link::buffered_link();
+		let (future, worker_sender) = BlockImportWorker::new(
+			result_sender,
+			verifier,
+			block_import,
+			justification_import,
+			finality_proof_import,
+		);
+
+		let mut pool = futures::executor::ThreadPool::builder()
+			.name_prefix("import-queue-worker-")
+			.pool_size(1)
+			.create()
+			.ok();
+
+		let manual_poll;
+		if let Some(pool) = &mut pool {
+			pool.spawn_ok(future);
+			manual_poll = None;
+		} else {
+			manual_poll = Some(Box::pin(future) as Pin<Box<_>>);
+		}
+
+		Self {
+			sender: worker_sender,
+			result_port,
+			manual_poll,
+			pool,
+		}
+	}
+}
+
+impl<B: BlockT> ImportQueue<B> for BasicQueue<B> {
+	fn import_blocks(&mut self, origin: BlockOrigin, blocks: Vec<IncomingBlock<B>>) {
+		if blocks.is_empty() {
+			return;
+		}
+
+		trace!(target: "sync", "Scheduling {} blocks for import", blocks.len());
+		let _ = self.sender.unbounded_send(ToWorkerMsg::ImportBlocks(origin, blocks));
+	}
+
+	fn import_justification(
+		&mut self,
+		who: Origin,
+		hash: B::Hash,
+		number: NumberFor<B>,
+		justification: Justification
+	) {
+		let _ = self.sender.unbounded_send(ToWorkerMsg::ImportJustification(who.clone(), hash, number, justification));
+	}
+
+	fn import_finality_proof(&mut self, who: Origin, hash: B::Hash, number: NumberFor<B>, finality_proof: Vec<u8>) {
+		trace!(target: "sync", "Scheduling finality proof of {}/{} for import", number, hash);
+		let _ = self.sender.unbounded_send(ToWorkerMsg::ImportFinalityProof(who, hash, number, finality_proof));
+	}
+
+	fn poll_actions(&mut self, cx: &mut Context, link: &mut dyn Link<B>) {
+		// As a backup mechanism, if we failed to spawn the `future_to_spawn`, we instead poll
+		// manually here.
+		if let Some(manual_poll) = self.manual_poll.as_mut() {
+			match Future::poll(Pin::new(manual_poll), cx) {
+				Poll::Pending => {}
+				_ => self.manual_poll = None,
+			}
+		}
+
+		self.result_port.poll_actions(cx, link);
+	}
+}
+
+/// Message destinated to the background worker.
+#[derive(Debug)]
+enum ToWorkerMsg<B: BlockT> {
+	ImportBlocks(BlockOrigin, Vec<IncomingBlock<B>>),
+	ImportJustification(Origin, B::Hash, NumberFor<B>, Justification),
+	ImportFinalityProof(Origin, B::Hash, NumberFor<B>, Vec<u8>),
+}
+
+struct BlockImportWorker<B: BlockT> {
+	result_sender: BufferedLinkSender<B>,
+	justification_import: Option<BoxJustificationImport<B>>,
+	finality_proof_import: Option<BoxFinalityProofImport<B>>,
+	delay_between_blocks: Duration,
+}
+
+impl<B: BlockT> BlockImportWorker<B> {
+	fn new<V: 'static + Verifier<B>>(
+		result_sender: BufferedLinkSender<B>,
+		verifier: V,
+		block_import: BoxBlockImport<B>,
+		justification_import: Option<BoxJustificationImport<B>>,
+		finality_proof_import: Option<BoxFinalityProofImport<B>>,
+	) -> (impl Future<Output = ()> + Send, mpsc::UnboundedSender<ToWorkerMsg<B>>) {
+		let (sender, mut port) = mpsc::unbounded();
+
+		let mut worker = BlockImportWorker {
+			result_sender,
+			justification_import,
+			finality_proof_import,
+			delay_between_blocks: Duration::new(0, 0),
+		};
+
+		// Let's initialize `justification_import` and `finality_proof_import`.
+		if let Some(justification_import) = worker.justification_import.as_mut() {
+			for (hash, number) in justification_import.on_start() {
+				worker.result_sender.request_justification(&hash, number);
+			}
+		}
+		if let Some(finality_proof_import) = worker.finality_proof_import.as_mut() {
+			for (hash, number) in finality_proof_import.on_start() {
+				worker.result_sender.request_finality_proof(&hash, number);
+			}
+		}
+
+		// The future below has two possible states:
+		//
+		// - Currently importing many blocks, in which case `importing` is `Some` and contains a
+		//   `Future`, and `block_import` is `None`.
+		// - Something else, in which case `block_import` is `Some` and `importing` is None.
+		//
+		let mut block_import_verifier = Some((block_import, verifier));
+		let mut importing = None;
+
+		let future = futures::future::poll_fn(move |cx| {
+			loop {
+				// If the results sender is closed, that means that the import queue is shutting
+				// down and we should end this future.
+				if worker.result_sender.is_closed() {
+					return Poll::Ready(())
+				}
+
+				// If we are in the process of importing a bunch of block, let's resume this
+				// process before doing anything more.
+				if let Some(imp_fut) = importing.as_mut() {
+					match Future::poll(Pin::new(imp_fut), cx) {
+						Poll::Pending => return Poll::Pending,
+						Poll::Ready((bi, verif)) => {
+							block_import_verifier = Some((bi, verif));
+							importing = None;
+						},
+					}
+				}
+
+				debug_assert!(importing.is_none());
+				debug_assert!(block_import_verifier.is_some());
+
+				// Grab the next action request sent to the import queue.
+				let msg = match Stream::poll_next(Pin::new(&mut port), cx) {
+					Poll::Ready(Some(msg)) => msg,
+					Poll::Ready(None) => return Poll::Ready(()),
+					Poll::Pending => return Poll::Pending,
+				};
+
+				match msg {
+					ToWorkerMsg::ImportBlocks(origin, blocks) => {
+						// On blocks import request, we merely *start* the process and store
+						// a `Future` into `importing`.
+						let (bi, verif) = block_import_verifier.take()
+							.expect("block_import_verifier is always Some; qed");
+						importing = Some(worker.import_a_batch_of_blocks(bi, verif, origin, blocks));
+					},
+					ToWorkerMsg::ImportFinalityProof(who, hash, number, proof) => {
+						let (_, verif) = block_import_verifier.as_mut()
+							.expect("block_import_verifier is always Some; qed");
+						worker.import_finality_proof(verif, who, hash, number, proof);
+					},
+					ToWorkerMsg::ImportJustification(who, hash, number, justification) => {
+						worker.import_justification(who, hash, number, justification);
+					}
+				}
+			}
+		});
+
+		(future, sender)
+	}
+
+	/// Returns a `Future` that imports the given blocks and sends the results on
+	/// `self.result_sender`.
+	///
+	/// For lifetime reasons, the `BlockImport` implementation must be passed by value, and is
+	/// yielded back in the output once the import is finished.
+	fn import_a_batch_of_blocks<V: 'static + Verifier<B>>(
+		&mut self,
+		block_import: BoxBlockImport<B>,
+		verifier: V,
+		origin: BlockOrigin,
+		blocks: Vec<IncomingBlock<B>>
+	) -> impl Future<Output = (BoxBlockImport<B>, V)> {
+		let mut result_sender = self.result_sender.clone();
+
+		import_many_blocks(block_import, origin, blocks, verifier, self.delay_between_blocks)
+			.then(move |(imported, count, results, block_import, verifier)| {
+				result_sender.blocks_processed(imported, count, results);
+				future::ready((block_import, verifier))
+			})
+	}
+
+	fn import_finality_proof<V: 'static + Verifier<B>>(
+		&mut self,
+		verifier: &mut V,
+		who: Origin,
+		hash: B::Hash,
+		number: NumberFor<B>,
+		finality_proof: Vec<u8>
+	) {
+		let result = self.finality_proof_import.as_mut().map(|finality_proof_import| {
+			finality_proof_import.import_finality_proof(hash, number, finality_proof, verifier)
+				.map_err(|e| {
+					debug!(
+						"Finality proof import failed with {:?} for hash: {:?} number: {:?} coming from node: {:?}",
+						e,
+						hash,
+						number,
+						who,
+					);
+				})
+		}).unwrap_or(Err(()));
+
+		trace!(target: "sync", "Imported finality proof for {}/{}", number, hash);
+		self.result_sender.finality_proof_imported(who, (hash, number), result);
+	}
+
+	fn import_justification(
+		&mut self,
+		who: Origin,
+		hash: B::Hash,
+		number: NumberFor<B>,
+		justification: Justification
+	) {
+		let success = self.justification_import.as_mut().map(|justification_import| {
+			justification_import.import_justification(hash, number, justification)
+				.map_err(|e| {
+					debug!(
+						target: "sync",
+						"Justification import failed with {:?} for hash: {:?} number: {:?} coming from node: {:?}",
+						e,
+						hash,
+						number,
+						who,
+					);
+					e
+				}).is_ok()
+		}).unwrap_or(false);
+
+		self.result_sender.justification_imported(who, &hash, number, success);
+	}
+}
+
+/// Import several blocks at once, returning import result for each block.
+///
+/// For lifetime reasons, the `BlockImport` implementation must be passed by value, and is yielded
+/// back in the output once the import is finished.
+///
+/// The returned `Future` yields at every imported block, which makes the execution more
+/// fine-grained and making it possible to interrupt the process.
+fn import_many_blocks<B: BlockT, V: Verifier<B>>(
+	import_handle: BoxBlockImport<B>,
+	blocks_origin: BlockOrigin,
+	blocks: Vec<IncomingBlock<B>>,
+	verifier: V,
+	delay_between_blocks: Duration,
+) -> impl Future<Output = (usize, usize, Vec<(
+	Result<BlockImportResult<NumberFor<B>>, BlockImportError>,
+	B::Hash,
+)>, BoxBlockImport<B>, V)> {
+	let count = blocks.len();
+
+	let blocks_range = match (
+		blocks.first().and_then(|b| b.header.as_ref().map(|h| h.number())),
+		blocks.last().and_then(|b| b.header.as_ref().map(|h| h.number())),
+	) {
+		(Some(first), Some(last)) if first != last => format!(" ({}..{})", first, last),
+		(Some(first), Some(_)) => format!(" ({})", first),
+		_ => Default::default(),
+	};
+
+	trace!(target: "sync", "Starting import of {} blocks {}", count, blocks_range);
+
+	let mut imported = 0;
+	let mut results = vec![];
+	let mut has_error = false;
+	let mut blocks = blocks.into_iter();
+	let mut import_handle = Some(import_handle);
+	let mut waiting = None;
+	let mut verifier = Some(verifier);
+
+	// Blocks in the response/drain should be in ascending order.
+
+	future::poll_fn(move |cx| {
+		// Handle the optional timer that makes us wait before the next import.
+		if let Some(waiting) = &mut waiting {
+			match Future::poll(Pin::new(waiting), cx) {
+				Poll::Ready(_) => {},
+				Poll::Pending => return Poll::Pending,
+			}
+		}
+		waiting = None;
+
+		// Is there any block left to import?
+		let block = match blocks.next() {
+			Some(b) => b,
+			None => {
+				// No block left to import, success!
+				let import_handle = import_handle.take()
+					.expect("Future polled again after it has finished");
+				let verifier = verifier.take()
+					.expect("Future polled again after it has finished");
+				let results = mem::replace(&mut results, Vec::new());
+				return Poll::Ready((imported, count, results, import_handle, verifier));
+			},
+		};
+
+		// We extract the content of `import_handle` and `verifier` only when the future ends,
+		// therefore `import_handle` and `verifier` are always `Some` here. It is illegal to poll
+		// a `Future` again after it has ended.
+		let import_handle = import_handle.as_mut()
+			.expect("Future polled again after it has finished");
+		let verifier = verifier.as_mut()
+			.expect("Future polled again after it has finished");
+
+		let block_number = block.header.as_ref().map(|h| h.number().clone());
+		let block_hash = block.hash;
+		let import_result = if has_error {
+			Err(BlockImportError::Cancelled)
+		} else {
+			// The actual import.
+			import_single_block(
+				&mut **import_handle,
+				blocks_origin.clone(),
+				block,
+				verifier,
+			)
+		};
+
+		if import_result.is_ok() {
+			trace!(target: "sync", "Block imported successfully {:?} ({})", block_number, block_hash);
+			imported += 1;
+		} else {
+			has_error = true;
+		}
+
+		results.push((import_result, block_hash));
+
+		// Notifies the current task again so that we re-execute this closure again for the next
+		// block.
+		if delay_between_blocks != Duration::new(0, 0) {
+			waiting = Some(Delay::new(delay_between_blocks));
+		}
+		cx.waker().wake_by_ref();
+		Poll::Pending
+	})
+}
@@ -0,0 +1,173 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Provides the `buffered_link` utility.
+//!
+//! The buffered link is a channel that allows buffering the method calls on `Link`.
+//!
+//! # Example
+//!
+//! ```
+//! use substrate_consensus_common::import_queue::Link;
+//! # use substrate_consensus_common::import_queue::buffered_link::buffered_link;
+//! # use test_client::runtime::Block;
+//! # struct DummyLink; impl Link<Block> for DummyLink {}
+//! # let mut my_link = DummyLink;
+//! let (mut tx, mut rx) = buffered_link::<Block>();
+//! tx.blocks_processed(0, 0, vec![]);
+//!
+//! // Calls `my_link.blocks_processed(0, 0, vec![])` when polled.
+//! let _fut = futures::future::poll_fn(move |cx| {
+//! 	rx.poll_actions(cx, &mut my_link);
+//! 	std::task::Poll::Pending::<()>
+//! });
+//! ```
+//!
+
+use futures::{prelude::*, channel::mpsc};
+use sr_primitives::traits::{Block as BlockT, NumberFor};
+use std::{pin::Pin, task::Context, task::Poll};
+use crate::import_queue::{Origin, Link, BlockImportResult, BlockImportError};
+
+/// Wraps around an unbounded channel from the `futures` crate. The sender implements `Link` and
+/// can be used to buffer commands, and the receiver can be used to poll said commands and transfer
+/// them to another link.
+pub fn buffered_link<B: BlockT>() -> (BufferedLinkSender<B>, BufferedLinkReceiver<B>) {
+	let (tx, rx) = mpsc::unbounded();
+	let tx = BufferedLinkSender { tx };
+	let rx = BufferedLinkReceiver { rx };
+	(tx, rx)
+}
+
+/// See [`buffered_link`].
+pub struct BufferedLinkSender<B: BlockT> {
+	tx: mpsc::UnboundedSender<BlockImportWorkerMsg<B>>,
+}
+
+impl<B: BlockT> BufferedLinkSender<B> {
+	/// Returns true if the sender points to nowhere.
+	///
+	/// Once `true` is returned, it is pointless to use the sender anymore.
+	pub fn is_closed(&self) -> bool {
+		self.tx.is_closed()
+	}
+}
+
+impl<B: BlockT> Clone for BufferedLinkSender<B> {
+	fn clone(&self) -> Self {
+		BufferedLinkSender {
+			tx: self.tx.clone(),
+		}
+	}
+}
+
+/// Internal buffered message.
+enum BlockImportWorkerMsg<B: BlockT> {
+	BlocksProcessed(usize, usize, Vec<(Result<BlockImportResult<NumberFor<B>>, BlockImportError>, B::Hash)>),
+	JustificationImported(Origin, B::Hash, NumberFor<B>, bool),
+	RequestJustification(B::Hash, NumberFor<B>),
+	FinalityProofImported(Origin, (B::Hash, NumberFor<B>), Result<(B::Hash, NumberFor<B>), ()>),
+	RequestFinalityProof(B::Hash, NumberFor<B>),
+}
+
+impl<B: BlockT> Link<B> for BufferedLinkSender<B> {
+	fn blocks_processed(
+		&mut self,
+		imported: usize,
+		count: usize,
+		results: Vec<(Result<BlockImportResult<NumberFor<B>>, BlockImportError>, B::Hash)>
+	) {
+		let _ = self.tx.unbounded_send(BlockImportWorkerMsg::BlocksProcessed(imported, count, results));
+	}
+
+	fn justification_imported(
+		&mut self,
+		who: Origin,
+		hash: &B::Hash,
+		number: NumberFor<B>,
+		success: bool
+	) {
+		let msg = BlockImportWorkerMsg::JustificationImported(who, hash.clone(), number, success);
+		let _ = self.tx.unbounded_send(msg);
+	}
+
+	fn request_justification(&mut self, hash: &B::Hash, number: NumberFor<B>) {
+		let _ = self.tx.unbounded_send(BlockImportWorkerMsg::RequestJustification(hash.clone(), number));
+	}
+
+	fn finality_proof_imported(
+		&mut self,
+		who: Origin,
+		request_block: (B::Hash, NumberFor<B>),
+		finalization_result: Result<(B::Hash, NumberFor<B>), ()>,
+	) {
+		let msg = BlockImportWorkerMsg::FinalityProofImported(who, request_block, finalization_result);
+		let _ = self.tx.unbounded_send(msg);
+	}
+
+	fn request_finality_proof(&mut self, hash: &B::Hash, number: NumberFor<B>) {
+		let _ = self.tx.unbounded_send(BlockImportWorkerMsg::RequestFinalityProof(hash.clone(), number));
+	}
+}
+
+/// See [`buffered_link`].
+pub struct BufferedLinkReceiver<B: BlockT> {
+	rx: mpsc::UnboundedReceiver<BlockImportWorkerMsg<B>>,
+}
+
+impl<B: BlockT> BufferedLinkReceiver<B> {
+	/// Polls for the buffered link actions. Any enqueued action will be propagated to the link
+	/// passed as parameter.
+	///
+	/// This method should behave in a way similar to `Future::poll`. It can register the current
+	/// task and notify later when more actions are ready to be polled. To continue the comparison,
+	/// it is as if this method always returned `Poll::Pending`.
+	pub fn poll_actions(&mut self, cx: &mut Context, link: &mut dyn Link<B>) {
+		loop {
+			let msg = if let Poll::Ready(Some(msg)) = Stream::poll_next(Pin::new(&mut self.rx), cx) {
+				msg
+			} else {
+				break
+			};
+
+			match msg {
+				BlockImportWorkerMsg::BlocksProcessed(imported, count, results) =>
+					link.blocks_processed(imported, count, results),
+				BlockImportWorkerMsg::JustificationImported(who, hash, number, success) =>
+					link.justification_imported(who, &hash, number, success),
+				BlockImportWorkerMsg::RequestJustification(hash, number) =>
+					link.request_justification(&hash, number),
+				BlockImportWorkerMsg::FinalityProofImported(who, block, result) =>
+					link.finality_proof_imported(who, block, result),
+				BlockImportWorkerMsg::RequestFinalityProof(hash, number) =>
+					link.request_finality_proof(&hash, number),
+			}
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use test_client::runtime::Block;
+
+	#[test]
+	fn is_closed() {
+		let (tx, rx) = super::buffered_link::<Block>();
+		assert!(!tx.is_closed());
+		drop(rx);
+		assert!(tx.is_closed());
+	}
+}