Batch signature verification (#5023)

* create parallel tasks extension * make type system happy * basic externalities * test for dynamic extensions * batching test * remove premature verify_batch * shnschnorrkel batch * alter test * shnschnorrkel test * executive batching * some docs * also multi/any signatgures * error propagation * styling * make verification extension optional * experimental ed25519 parallelization * some merge fallout * utilize task executor * merge fallout * utilize task executor more * another merge fallout * feature-gate sp-io * arrange toml * fix no-std * sr25519 batching and refactoring * add docs * fix name * add newline * fix block import test * long sr25519 test * blocking instead of parking * move everything in crypto * return batch_verify to check :) * use condvars * use multi-threaded executor for benches * don't call via host interface * try no spawning * add true * cleanup * straighten batching * remove signature check from this test (?) * remove now pointless test * remove another now useless test * fix warnings * Revert "remove another now useless test" This reverts commit bbdec24bb67ed4373072daef7c863e1a8825bd8b. * rethink the sp-io-part * Revert "remove now pointless test" This reverts commit 4d553066322e65782264caa6053d4cd5538df977. * fix wording * add wording * add todo and fix * return check and fix * add logging in sp-io * Update primitives/io/src/batch_verifier.rs Co-Authored-By: cheme <emericchevalier.pro@gmail.com> * address review and use std condvar * account for early exit * address reivew * address review * more suggestions * add docs for batch verification * remove unused * more review suggestions * move to sp-runtime * add expects * remove blocks * use entry * Update primitives/io/src/batch_verifier.rs Co-Authored-By: Bastian Köcher <bkchr@users.noreply.github.com> * Update primitives/externalities/src/extensions.rs Co-Authored-By: Bastian Köcher <bkchr@users.noreply.github.com> * update overlooked note * remove stupid return * Update primitives/io/src/lib.rs Co-Authored-By: Bastian Köcher <bkchr@users.noreply.github.com> * Update primitives/io/src/lib.rs Co-Authored-By: Bastian Köcher <bkchr@users.noreply.github.com> * fix wording * bump spec_version Co-authored-by: cheme <emericchevalier.pro@gmail.com> Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
2026-04-26 15:47:58 +00:00 · 2020-04-16 22:40:04 +03:00
parent 36243068bd
commit 372f8b2c7e
21 changed files with 711 additions and 47 deletions
@@ -0,0 +1,163 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Batch/parallel verification.
+
+use sp_core::{ed25519, sr25519, crypto::Pair, traits::CloneableSpawn};
+use std::sync::{Arc, atomic::{AtomicBool, Ordering as AtomicOrdering}};
+use futures::{future::FutureExt, task::FutureObj, channel::oneshot};
+
+#[derive(Debug, Clone)]
+struct Sr25519BatchItem {
+	signature: sr25519::Signature,
+	pub_key: sr25519::Public,
+	message: Vec<u8>,
+}
+
+/// Batch verifier.
+///
+/// Used to parallel-verify signatures for runtime host. Provide task executor and
+/// just push (`push_ed25519`, `push_sr25519`) as many signature as you need. At the end,
+/// call `verify_and_clear to get a result. After that, batch verifier is ready for the
+/// next batching job.
+pub struct BatchVerifier {
+	scheduler: Box<dyn CloneableSpawn>,
+	sr25519_items: Vec<Sr25519BatchItem>,
+	invalid: Arc<AtomicBool>,
+	pending_tasks: Vec<oneshot::Receiver<()>>,
+}
+
+impl BatchVerifier {
+	pub fn new(scheduler: Box<dyn CloneableSpawn>) -> Self {
+		BatchVerifier {
+			scheduler,
+			sr25519_items: Default::default(),
+			invalid: Arc::new(false.into()),
+			pending_tasks: vec![],
+		}
+	}
+
+	fn spawn_verification_task(
+		&mut self, f: impl FnOnce() -> bool + Send + 'static,
+	) -> Result<(), ()> {
+		// there is already invalid transaction encountered
+		if self.invalid.load(AtomicOrdering::Relaxed) { return Err(()); }
+
+		let invalid_clone = self.invalid.clone();
+		let (sender, receiver) = oneshot::channel();
+		self.pending_tasks.push(receiver);
+
+		self.scheduler.spawn_obj(FutureObj::new(async move {
+			if !f() {
+				invalid_clone.store(true, AtomicOrdering::Relaxed);
+			}
+			if sender.send(()).is_err() {
+				// sanity
+				log::warn!("Verification halted while result was pending");
+				invalid_clone.store(true, AtomicOrdering::Relaxed);
+			}
+		}.boxed())).map_err(drop)
+	}
+
+	/// Push ed25519 signature to verify.
+	///
+	/// Returns false if some of the pushed signatures before already failed the check
+	/// (in this case it won't verify anything else)
+	pub fn push_ed25519(
+		&mut self,
+		signature: ed25519::Signature,
+		pub_key: ed25519::Public,
+		message: Vec<u8>,
+	) -> bool {
+		if self.invalid.load(AtomicOrdering::Relaxed) { return false; }
+
+		if self.spawn_verification_task(move || ed25519::Pair::verify(&signature, &message, &pub_key)).is_err() {
+			log::debug!(
+				target: "runtime",
+				"Batch-verification returns false because failed to spawn background task.",
+			);
+
+			return false;
+		}
+		true
+	}
+
+	/// Push sr25519 signature to verify.
+	///
+	/// Returns false if some of the pushed signatures before already failed the check.
+	/// (in this case it won't verify anything else)
+	pub fn push_sr25519(
+		&mut self,
+		signature: sr25519::Signature,
+		pub_key: sr25519::Public,
+		message: Vec<u8>,
+	) -> bool {
+		if self.invalid.load(AtomicOrdering::Relaxed) { return false; }
+		self.sr25519_items.push(Sr25519BatchItem { signature, pub_key, message });
+		true
+	}
+
+	/// Verify all previously pushed signatures since last call and return
+	/// aggregated result.
+	#[must_use]
+	pub fn verify_and_clear(&mut self) -> bool {
+		use std::sync::{Mutex, Condvar};
+
+		let pending = std::mem::replace(&mut self.pending_tasks, vec![]);
+
+		log::trace!(
+			target: "runtime",
+			"Batch-verification: {} pending tasks, {} sr25519 signatures",
+			pending.len(),
+			self.sr25519_items.len(),
+		);
+
+		let messages = self.sr25519_items.iter().map(|item| &item.message[..]).collect();
+		let signatures = self.sr25519_items.iter().map(|item| &item.signature).collect();
+		let pub_keys = self.sr25519_items.iter().map(|item| &item.pub_key).collect();
+
+		if !sr25519::verify_batch(messages, signatures, pub_keys) {
+			self.sr25519_items.clear();
+
+			return false;
+		}
+
+		self.sr25519_items.clear();
+
+		if pending.len() > 0 {
+			let pair = Arc::new((Mutex::new(()), Condvar::new()));
+			let pair_clone = pair.clone();
+
+			if self.scheduler.spawn_obj(FutureObj::new(async move {
+				futures::future::join_all(pending).await;
+				pair_clone.1.notify_all();
+			}.boxed())).is_err() {
+				log::debug!(
+					target: "runtime",
+					"Batch-verification returns false because failed to spawn background task.",
+				);
+
+				return false;
+			}
+
+			let (mtx, cond_var) = &*pair;
+			let mtx = mtx.lock().expect("Locking can only fail when the mutex is poisoned; qed");
+			let _ = cond_var.wait(mtx).expect("Waiting can only fail when the mutex waited on is poisoned; qed");
+		}
+
+		!self.invalid.swap(false, AtomicOrdering::Relaxed)
+	}
+}
@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.

-//! This is part of the Substrate runtime.
+//! I/O host interface for substrate runtime.

 #![warn(missing_docs)]

@@ -34,7 +34,7 @@ use sp_std::ops::Deref;
 #[cfg(feature = "std")]
 use sp_core::{
 	crypto::Pair,
-	traits::{KeystoreExt, CallInWasmExt},
+	traits::{KeystoreExt, CallInWasmExt, TaskExecutorExt},
 	offchain::{OffchainExt, TransactionPoolExt},
 	hexdisplay::HexDisplay,
 	storage::{ChildStorageKey, ChildInfo},
@@ -57,6 +57,12 @@ use codec::{Encode, Decode};
 #[cfg(feature = "std")]
 use sp_externalities::{ExternalitiesExt, Externalities};

+#[cfg(feature = "std")]
+mod batch_verifier;
+
+#[cfg(feature = "std")]
+use batch_verifier::BatchVerifier;
+
 /// Error verifying ECDSA signature
 #[derive(Encode, Decode)]
 pub enum EcdsaVerifyError {
@@ -416,16 +422,98 @@ pub trait Crypto {
 			.ok()
 	}

-	/// Verify an `ed25519` signature.
+	/// Verify `ed25519` signature.
 	///
-	/// Returns `true` when the verification in successful.
+	/// Returns `true` when the verification is either successful or batched.
+	/// If no batching verification extension registered, this will return the result
+	/// of verification immediately. If batching verification extension is registered
+	/// caller should call `crypto::finish_batch_verify` to actualy check all submitted
+	/// signatures.
 	fn ed25519_verify(
-		&self,
 		sig: &ed25519::Signature,
 		msg: &[u8],
 		pub_key: &ed25519::Public,
 	) -> bool {
-		ed25519::Pair::verify(sig, msg, pub_key)
+		// TODO: see #5554, this is used outside of externalities context/runtime, thus this manual
+		// `with_externalities`.
+		//
+		// This `with_externalities(..)` block returns Some(Some(result)) if signature verification was successfully
+		// batched, everything else (Some(None)/None) means it was not batched and needs to be verified.
+		let evaluated = sp_externalities::with_externalities(|mut instance|
+			instance.extension::<VerificationExt>().map(
+				|extension| extension.push_ed25519(
+					sig.clone(),
+					pub_key.clone(),
+					msg.to_vec(),
+				)
+			)
+		);
+
+		match evaluated {
+			Some(Some(val)) => val,
+			_ => ed25519::Pair::verify(sig, msg, pub_key),
+		}
+	}
+
+	/// Verify `sr25519` signature.
+	///
+	/// Returns `true` when the verification is either successful or batched.
+	/// If no batching verification extension registered, this will return the result
+	/// of verification immediately. If batching verification extension is registered,
+	/// caller should call `crypto::finish_batch_verify` to actualy check all submitted
+	#[version(2)]
+	fn sr25519_verify(
+		sig: &sr25519::Signature,
+		msg: &[u8],
+		pub_key: &sr25519::Public,
+	) -> bool {
+		// TODO: see #5554, this is used outside of externalities context/runtime, thus this manual
+		// `with_externalities`.
+		//
+		// This `with_externalities(..)` block returns Some(Some(result)) if signature verification was successfully
+		// batched, everything else (Some(None)/None) means it was not batched and needs to be verified.
+		let evaluated = sp_externalities::with_externalities(|mut instance|
+			instance.extension::<VerificationExt>().map(
+				|extension| extension.push_sr25519(
+					sig.clone(),
+					pub_key.clone(),
+					msg.to_vec(),
+				)
+			)
+		);
+
+		match evaluated {
+			Some(Some(val)) => val,
+			_ => sr25519::Pair::verify(sig, msg, pub_key),
+		}
+	}
+
+	/// Start verification extension.
+	fn start_batch_verify(&mut self) {
+		let scheduler = self.extension::<TaskExecutorExt>()
+			.expect("No task executor associated with the current context!")
+			.0
+			.clone();
+
+		self.register_extension(VerificationExt(BatchVerifier::new(scheduler)))
+			.expect("Failed to register required extension: `VerificationExt`");
+	}
+
+	/// Finish batch-verification of signatures.
+	///
+	/// Verify or wait for verification to finish for all signatures which were previously
+	/// deferred by `sr25519_verify`/`ed25519_verify`.
+	///
+	/// Will panic if no `VerificationExt` is registered (`start_batch_verify` was not called).
+	fn finish_batch_verify(&mut self) -> bool {
+		let result = self.extension::<VerificationExt>()
+			.expect("`finish_batch_verify` should only be called after `start_batch_verify`")
+			.verify_and_clear();
+
+		self.deregister_extension::<VerificationExt>()
+			.expect("No verification extension in current context!");
+
+		result
 	}

 	/// Returns all `sr25519` public keys for the given key id from the keystore.
@@ -477,14 +565,6 @@ pub trait Crypto {
 		sr25519::Pair::verify_deprecated(sig, msg, pubkey)
 	}

-	/// Verify an `sr25519` signature.
-	///
-	/// Returns `true` when the verification in successful.
-	#[version(2)]
-	fn sr25519_verify(sig: &sr25519::Signature, msg: &[u8], pubkey: &sr25519::Public) -> bool {
-		sr25519::Pair::verify(sig, msg, pubkey)
-	}
-
 	/// Verify and recover a SECP256k1 ECDSA signature.
 	///
 	/// - `sig` is passed in RSV format. V should be either `0/1` or `27/28`.
@@ -566,6 +646,12 @@ pub trait Hashing {
 	}
 }

+#[cfg(feature = "std")]
+sp_externalities::decl_extension! {
+	/// The keystore extension to register/retrieve from the externalities.
+	pub struct VerificationExt(BatchVerifier);
+}
+
 /// Interface that provides functions to access the offchain functionality.
 #[runtime_interface]
 pub trait Offchain {
@@ -949,6 +1035,7 @@ mod tests {
 	use sp_core::map;
 	use sp_state_machine::BasicExternalities;
 	use sp_core::storage::Storage;
+	use std::any::TypeId;

 	#[test]
 	fn storage_works() {
@@ -1010,4 +1097,132 @@ mod tests {
 			assert!(storage::get(b":abc").is_none());
 		});
 	}
+
+	#[test]
+	fn dynamic_extensions_work() {
+		let mut ext = BasicExternalities::with_tasks_executor();
+		ext.execute_with(|| {
+			crypto::start_batch_verify();
+		});
+
+		assert!(ext.extensions().get_mut(TypeId::of::<VerificationExt>()).is_some());
+
+		ext.execute_with(|| {
+			crypto::finish_batch_verify();
+		});
+
+		assert!(ext.extensions().get_mut(TypeId::of::<VerificationExt>()).is_none());
+	}
+
+	#[test]
+	fn long_sr25519_batching() {
+		let mut ext = BasicExternalities::with_tasks_executor();
+		ext.execute_with(|| {
+			let pair = sr25519::Pair::generate_with_phrase(None).0;
+			crypto::start_batch_verify();
+			for it in 0..70 {
+				let msg = format!("Schnorrkel {}!", it);
+				let signature = pair.sign(msg.as_bytes());
+				crypto::sr25519_verify(&signature, msg.as_bytes(), &pair.public());
+			}
+
+			// push invlaid
+			crypto::sr25519_verify(
+				&Default::default(),
+				&Vec::new(),
+				&Default::default(),
+			);
+			assert!(!crypto::finish_batch_verify());
+
+			crypto::start_batch_verify();
+			for it in 0..70 {
+				let msg = format!("Schnorrkel {}!", it);
+				let signature = pair.sign(msg.as_bytes());
+				crypto::sr25519_verify(&signature, msg.as_bytes(), &pair.public());
+			}
+			assert!(crypto::finish_batch_verify());
+		});
+	}
+
+	#[test]
+	fn batching_works() {
+		let mut ext = BasicExternalities::with_tasks_executor();
+		ext.execute_with(|| {
+			// invalid ed25519 signature
+			crypto::start_batch_verify();
+			crypto::ed25519_verify(
+				&Default::default(),
+				&Vec::new(),
+				&Default::default(),
+			);
+			assert!(!crypto::finish_batch_verify());
+
+			// 2 valid ed25519 signatures
+			crypto::start_batch_verify();
+
+			let pair = ed25519::Pair::generate_with_phrase(None).0;
+			let msg = b"Important message";
+			let signature = pair.sign(msg);
+			crypto::ed25519_verify(&signature, msg, &pair.public());
+
+			let pair = ed25519::Pair::generate_with_phrase(None).0;
+			let msg = b"Even more important message";
+			let signature = pair.sign(msg);
+			crypto::ed25519_verify(&signature, msg, &pair.public());
+
+			assert!(crypto::finish_batch_verify());
+
+			// 1 valid, 1 invalid ed25519 signature
+			crypto::start_batch_verify();
+
+			let pair = ed25519::Pair::generate_with_phrase(None).0;
+			let msg = b"Important message";
+			let signature = pair.sign(msg);
+			crypto::ed25519_verify(&signature, msg, &pair.public());
+
+			crypto::ed25519_verify(
+				&Default::default(),
+				&Vec::new(),
+				&Default::default(),
+			);
+
+			assert!(!crypto::finish_batch_verify());
+
+			// 1 valid ed25519, 2 valid sr25519
+			crypto::start_batch_verify();
+
+			let pair = ed25519::Pair::generate_with_phrase(None).0;
+			let msg = b"Ed25519 batching";
+			let signature = pair.sign(msg);
+			crypto::ed25519_verify(&signature, msg, &pair.public());
+
+			let pair = sr25519::Pair::generate_with_phrase(None).0;
+			let msg = b"Schnorrkel rules";
+			let signature = pair.sign(msg);
+			crypto::sr25519_verify(&signature, msg, &pair.public());
+
+			let pair = sr25519::Pair::generate_with_phrase(None).0;
+			let msg = b"Schnorrkel batches!";
+			let signature = pair.sign(msg);
+			crypto::sr25519_verify(&signature, msg, &pair.public());
+
+			assert!(crypto::finish_batch_verify());
+
+			// 1 valid sr25519, 1 invalid sr25519
+			crypto::start_batch_verify();
+
+			let pair = sr25519::Pair::generate_with_phrase(None).0;
+			let msg = b"Schnorrkcel!";
+			let signature = pair.sign(msg);
+			crypto::sr25519_verify(&signature, msg, &pair.public());
+
+			crypto::sr25519_verify(
+				&Default::default(),
+				&Vec::new(),
+				&Default::default(),
+			);
+
+			assert!(!crypto::finish_batch_verify());
+		});
+	}
 }