Batch signature verification (#5023)

* create parallel tasks extension

* make type system happy

* basic externalities

* test for dynamic extensions

* batching test

* remove premature verify_batch

* shnschnorrkel batch

* alter test

* shnschnorrkel test

* executive batching

* some docs

* also multi/any signatgures

* error propagation

* styling

* make verification extension optional

* experimental ed25519 parallelization

* some merge fallout

* utilize task executor

* merge fallout

* utilize task executor more

* another merge fallout

* feature-gate sp-io

* arrange toml

* fix no-std

* sr25519 batching and refactoring

* add docs

* fix name

* add newline

* fix block import test

* long sr25519 test

* blocking instead of parking

* move everything in crypto

* return batch_verify to check :)

* use condvars

* use multi-threaded executor for benches

* don't call via host interface

* try no spawning

* add true

* cleanup

* straighten batching

* remove signature check from this test (?)

* remove now pointless test

* remove another now useless test

* fix warnings

* Revert "remove another now useless test"

This reverts commit bbdec24bb67ed4373072daef7c863e1a8825bd8b.

* rethink the sp-io-part

* Revert "remove now pointless test"

This reverts commit 4d553066322e65782264caa6053d4cd5538df977.

* fix wording

* add  wording

* add todo and fix

* return check and fix

* add logging in sp-io

* Update primitives/io/src/batch_verifier.rs

Co-Authored-By: cheme <emericchevalier.pro@gmail.com>

* address review and use std condvar

* account for early exit

* address reivew

* address review

* more suggestions

* add docs for batch verification

* remove unused

* more review suggestions

* move to sp-runtime

* add expects

* remove blocks

* use entry

* Update primitives/io/src/batch_verifier.rs

Co-Authored-By: Bastian Köcher <bkchr@users.noreply.github.com>

* Update primitives/externalities/src/extensions.rs

Co-Authored-By: Bastian Köcher <bkchr@users.noreply.github.com>

* update overlooked note

* remove stupid return

* Update primitives/io/src/lib.rs

Co-Authored-By: Bastian Köcher <bkchr@users.noreply.github.com>

* Update primitives/io/src/lib.rs

Co-Authored-By: Bastian Köcher <bkchr@users.noreply.github.com>

* fix wording

* bump spec_version

Co-authored-by: cheme <emericchevalier.pro@gmail.com>
Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

substrate/primitives/state-machine/src/basic.rs

@@ -32,17 +32,35 @@ use sp_core::{
 };
 use log::warn;
 use codec::Encode;
+use sp_externalities::Extensions;
 
 /// Simple Map-based Externalities impl.
 #[derive(Debug)]
 pub struct BasicExternalities {
 	inner: Storage,
+	extensions: Extensions,
 }
 
 impl BasicExternalities {
 	/// Create a new instance of `BasicExternalities`
 	pub fn new(inner: Storage) -> Self {
-		BasicExternalities { inner }
+		BasicExternalities { inner, extensions: Default::default() }
+	}
+
+	/// New basic externalities with empty storage.
+	pub fn new_empty() -> Self {
+		Self::new(Storage::default())
+	}
+
+	/// New basic extternalities with tasks executor.
+	pub fn with_tasks_executor() -> Self {
+		let mut extensions = Extensions::default();
+		extensions.register(sp_core::traits::TaskExecutorExt(sp_core::tasks::executor()));
+
+		Self {
+			inner: Storage::default(),
+			extensions,
+		}
 	}
 
 	/// Insert key/value
@@ -62,10 +80,13 @@ impl BasicExternalities {
 		storage: &mut sp_core::storage::Storage,
 		f: impl FnOnce() -> R,
 	) -> R {
-		let mut ext = Self { inner: Storage {
-			top: std::mem::replace(&mut storage.top, Default::default()),
-			children: std::mem::replace(&mut storage.children, Default::default()),
-		}};
+		let mut ext = Self {
+			inner: Storage {
+				top: std::mem::replace(&mut storage.top, Default::default()),
+				children: std::mem::replace(&mut storage.children, Default::default()),
+			},
+			extensions: Default::default(),
+		};
 
 		let r = ext.execute_with(f);
 
@@ -80,6 +101,11 @@ impl BasicExternalities {
 	pub fn execute_with<R>(&mut self, f: impl FnOnce() -> R) -> R {
 		sp_externalities::set_and_run_with_externalities(self, f)
 	}
+
+	/// List of active extensions.
+	pub fn extensions(&mut self) -> &mut Extensions {
+		&mut self.extensions
+	}
 }
 
 impl PartialEq for BasicExternalities {
@@ -103,10 +129,13 @@ impl Default for BasicExternalities {
 
 impl From<BTreeMap<StorageKey, StorageValue>> for BasicExternalities {
 	fn from(hashmap: BTreeMap<StorageKey, StorageValue>) -> Self {
-		BasicExternalities { inner: Storage {
-			top: hashmap,
-			children: Default::default(),
-		}}
+		BasicExternalities {
+			inner: Storage {
+				top: hashmap,
+				children: Default::default(),
+			},
+			extensions: Default::default(),
+		}
 	}
 }
 
@@ -279,9 +308,23 @@ impl Externalities for BasicExternalities {
 }
 
 impl sp_externalities::ExtensionStore for BasicExternalities {
-	fn extension_by_type_id(&mut self, _: TypeId) -> Option<&mut dyn Any> {
-		warn!("Extensions are not supported by `BasicExternalities`.");
-		None
+	fn extension_by_type_id(&mut self, type_id: TypeId) -> Option<&mut dyn Any> {
+		self.extensions.get_mut(type_id)
+	}
+
+	fn register_extension_with_type_id(
+		&mut self,
+		type_id: TypeId,
+		extension: Box<dyn sp_externalities::Extension>,
+	) -> Result<(), sp_externalities::Error> {
+		self.extensions.register_with_type_id(type_id, extension)
+	}
+
+	fn deregister_extension_by_type_id(&mut self, type_id: TypeId) -> Result<(), sp_externalities::Error> {
+		self.extensions
+			.deregister(type_id)
+			.ok_or(sp_externalities::Error::ExtensionIsNotRegistered(type_id))
+			.map(drop)
 	}
 }
 
@@ -347,7 +390,7 @@ mod tests {
 	#[test]
 	fn basic_externalities_is_empty() {
 		// Make sure no values are set by default in `BasicExternalities`.
-		let storage = BasicExternalities::new(Default::default()).into_storages();
+		let storage = BasicExternalities::new_empty().into_storages();
 		assert!(storage.top.is_empty());
 		assert!(storage.children.is_empty());
 	}

substrate/primitives/state-machine/src/ext.rs

@@ -28,7 +28,7 @@ use sp_core::{
 	traits::Externalities, hexdisplay::HexDisplay,
 };
 use sp_trie::{trie_types::Layout, default_child_trie_root};
-use sp_externalities::Extensions;
+use sp_externalities::{Extensions, Extension};
 use codec::{Decode, Encode};
 
 use std::{error, fmt, any::{Any, TypeId}};
@@ -548,6 +548,29 @@ where
 	fn extension_by_type_id(&mut self, type_id: TypeId) -> Option<&mut dyn Any> {
 		self.extensions.as_mut().and_then(|exts| exts.get_mut(type_id))
 	}
+
+	fn register_extension_with_type_id(
+		&mut self,
+		type_id: TypeId,
+		extension: Box<dyn Extension>,
+	) -> Result<(), sp_externalities::Error> {
+		if let Some(ref mut extensions) = self.extensions {
+			extensions.register_with_type_id(type_id, extension)
+		} else {
+			Err(sp_externalities::Error::ExtensionsAreNotSupported)
+		}
+	}
+
+	fn deregister_extension_by_type_id(&mut self, type_id: TypeId) -> Result<(), sp_externalities::Error> {
+		if let Some(ref mut extensions) = self.extensions {
+			match extensions.deregister(type_id) {
+				Some(_) => Ok(()),
+				None => Err(sp_externalities::Error::ExtensionIsNotRegistered(type_id))
+			}
+		} else {
+			Err(sp_externalities::Error::ExtensionsAreNotSupported)
+		}
+	}
 }
 
 #[cfg(test)]

substrate/primitives/state-machine/src/testing.rs

@@ -80,6 +80,11 @@ impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N>
 		Self::new_with_code(&[], storage)
 	}
 
+	/// New empty test externalities.
+	pub fn new_empty() -> Self {
+		Self::new_with_code(&[], Storage::default())
+	}
+
 	/// Create a new instance of `TestExternalities` with code and storage.
 	pub fn new_with_code(code: &[u8], mut storage: Storage) -> Self {
 		let mut overlay = OverlayedChanges::default();
@@ -93,12 +98,15 @@ impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N>
 		storage.top.insert(HEAP_PAGES.to_vec(), 8u64.encode());
 		storage.top.insert(CODE.to_vec(), code.to_vec());
 
+		let mut extensions = Extensions::default();
+		extensions.register(sp_core::traits::TaskExecutorExt(sp_core::tasks::executor()));
+
 		TestExternalities {
 			overlay,
 			changes_trie_config,
+			extensions,
 			changes_trie_storage: ChangesTrieInMemoryStorage::new(),
 			backend: storage.into(),
-			extensions: Default::default(),
 			storage_transaction_cache: Default::default(),
 		}
 	}
@@ -191,6 +199,21 @@ impl<H, N> sp_externalities::ExtensionStore for TestExternalities<H, N> where
 	fn extension_by_type_id(&mut self, type_id: TypeId) -> Option<&mut dyn Any> {
 		self.extensions.get_mut(type_id)
 	}
+
+	fn register_extension_with_type_id(
+		&mut self,
+		type_id: TypeId,
+		extension: Box<dyn Extension>,
+	) -> Result<(), sp_externalities::Error> {
+		self.extensions.register_with_type_id(type_id, extension)
+	}
+
+	fn deregister_extension_by_type_id(&mut self, type_id: TypeId) -> Result<(), sp_externalities::Error> {
+		self.extensions
+			.deregister(type_id)
+			.expect("There should be an extension we try to remove in TestExternalities");
+		Ok(())
+	}
 }
 
 #[cfg(test)]