Support for multiple signature scheme for BEEFY primitves (#14373)

* Merged BEEFY primitives with generic signature and keyset commitment support from old pull to current code

* - Add bls-experimental feature to application-crypto and beefy primitives
- Fix remaining crypto -> ecdsa_crypto
- code build but not tests

* Make beefy primitive tests compile

* move bls related beefy primitives code and test behind bls-experimental flag

* Make BEEFY clients complies with BEEFY API depending on AuthorityId

* - Rename `BeefyAuthoritySet.root` → `BeefyAuthoritySet.keyset_commitment`.
- Remove apk proof keyset_commitment from `BeefyAuthoritySet`.
- Fix failing signed commitment and signature to witness test.
- Make client compatible with BeefyAPI generic on AuthorityId.
- `crypto` → `ecdsa_crypto` in BEEFY client and frame.

* Commit Cargo lock remove ark-serialize from BEEFY primitives

* Use Codec instead of Encode + Decode in primitives/consensus/beefy/src/lib.rs

Co-authored-by: Davide Galassi <davxy@datawok.net>

* - Make `BeefyApi` generic over Signature type.
- Make new `BeeyApi` functinos also generic over AuthorityId and Signature

* Unmake BeefyAPI generic over Signature. Recover Signature type from AuthId.

* - dont use hex or hex-literal use array-bytes instead in beefy primitives and bls crypto.
- CamelCase ECDSA and BLS everywhere.

* Move the definition of BEEFY key type from `primitives/beefy` to `crypto.rs` according to new convention.

* - Add bls377_generate_new to `sp-io` and `application_crypto::bls`.
- Add `bls-experimental` to `sp-io`

Does not compile because PassByCodec can not derive PassBy using customly implemented PassByIner.

* Implement PassBy for `bls::Public` manually

* fix Beefy `KEY_TYPE` in `frame/beefy` tests to come from `sp-core::key_types` enum

* specify both generic for `hex2array_unchecked` in `sp-core/bls.rs`

* Rename `crypto`→`ecdsa_crypto` in `primitives/consensus/beefy/src/test_utils.rs` docs

* remove commented-out code in `primitives/consensus/beefy/src/commitment.rs`

Co-authored-by: Davide Galassi <davxy@datawok.net>

* Fix inconsistency in panic message in  `primitives/io/src/lib.rs`

Co-authored-by: Davide Galassi <davxy@datawok.net>

* Remove redundant feature activation in `primitives/io/Cargo.toml`

Co-authored-by: Davide Galassi <davxy@datawok.net>

* - make `w3f-bls` a dev-dependancy only for beefy primitives.

- clean up comments.

Co-authored-by: Davide Galassi <davxy@datawok.net>

* export BEEFY KEY_TYPE from primitives/consensus/beefy
make `frame/consensus/beefy` in dependent of sp_crypto_app
use consistent naming in the beefy primitive tests.

* - implement `BeefyAuthorityId` for `bls_crypto::AuthorityId`.
- implement `bls_verify_works` test for BEEFY `bls_crypto`.

* Remove BEEFY `ecdsa_n_bls_crypto` for now for later re-introduction

* Make commitment and witness BEEFY tests not use Keystore.

* put `bls_beefy_verify_works` test under `bls-experimental` flag.

* bump up Runtime `BeefyAPI` to version 3 due to introducing generic AuthorityId.

* reuse code and encapsulate w3f-bls backend in sp-core as most as possible

Co-authored-by: Davide Galassi <davxy@datawok.net>

* Make comments in primities BEEFY `commitment.rs` and `witness.rs``tests convention conforming

* Use master dep versions

* Trivial change. Mostly to trigger CI

* Apply suggestions from code review

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

* Fix Cargo.toml

* Trigger CI with cumulus companion

* Trigger CI after polkadot companion change

---------

Co-authored-by: Davide Galassi <davxy@datawok.net>
Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

substrate/client/consensus/beefy/src/communication/gossip.rs

@@ -41,7 +41,7 @@ use crate::{
 	LOG_TARGET,
 };
 use sp_consensus_beefy::{
-	crypto::{AuthorityId, Signature},
+	ecdsa_crypto::{AuthorityId, Signature},
 	ValidatorSet, ValidatorSetId, VoteMessage,
 };
 
@@ -476,9 +476,10 @@ pub(crate) mod tests {
 	use super::*;
 	use crate::keystore::BeefyKeystore;
 	use sc_network_test::Block;
+	use sp_application_crypto::key_types::BEEFY as BEEFY_KEY_TYPE;
 	use sp_consensus_beefy::{
-		crypto::Signature, known_payloads, Commitment, Keyring, MmrRootHash, Payload,
-		SignedCommitment, VoteMessage, KEY_TYPE,
+		ecdsa_crypto::Signature, known_payloads, Commitment, Keyring, MmrRootHash, Payload,
+		SignedCommitment, VoteMessage,
 	};
 	use sp_keystore::{testing::MemoryKeystore, Keystore};
 
@@ -536,7 +537,7 @@ pub(crate) mod tests {
 
 	pub fn sign_commitment<BN: Encode>(who: &Keyring, commitment: &Commitment<BN>) -> Signature {
 		let store = MemoryKeystore::new();
-		store.ecdsa_generate_new(KEY_TYPE, Some(&who.to_seed())).unwrap();
+		store.ecdsa_generate_new(BEEFY_KEY_TYPE, Some(&who.to_seed())).unwrap();
 		let beefy_keystore: BeefyKeystore = Some(store.into()).into();
 		beefy_keystore.sign(&who.public(), &commitment.encode()).unwrap()
 	}

substrate/client/consensus/beefy/src/communication/request_response/outgoing_requests_engine.rs

@@ -26,7 +26,7 @@ use sc_network::{
 	request_responses::{IfDisconnected, RequestFailure},
 	NetworkRequest, PeerId, ProtocolName,
 };
-use sp_consensus_beefy::{crypto::AuthorityId, ValidatorSet};
+use sp_consensus_beefy::{ecdsa_crypto::AuthorityId, ValidatorSet};
 use sp_runtime::traits::{Block, NumberFor};
 use std::{collections::VecDeque, result::Result, sync::Arc};
 

substrate/client/consensus/beefy/src/import.rs

@@ -22,7 +22,7 @@ use log::debug;
 
 use sp_api::{ProvideRuntimeApi, TransactionFor};
 use sp_consensus::Error as ConsensusError;
-use sp_consensus_beefy::{BeefyApi, BEEFY_ENGINE_ID};
+use sp_consensus_beefy::{ecdsa_crypto::AuthorityId, BeefyApi, BEEFY_ENGINE_ID};
 use sp_runtime::{
 	traits::{Block as BlockT, Header as HeaderT, NumberFor},
 	EncodedJustification,
@@ -83,7 +83,7 @@ where
 	Block: BlockT,
 	BE: Backend<Block>,
 	Runtime: ProvideRuntimeApi<Block>,
-	Runtime::Api: BeefyApi<Block> + Send,
+	Runtime::Api: BeefyApi<Block, AuthorityId> + Send,
 {
 	fn decode_and_verify(
 		&self,
@@ -125,7 +125,7 @@ where
 		> + Send
 		+ Sync,
 	Runtime: ProvideRuntimeApi<Block> + Send + Sync,
-	Runtime::Api: BeefyApi<Block>,
+	Runtime::Api: BeefyApi<Block, AuthorityId>,
 {
 	type Error = ConsensusError;
 	type Transaction = TransactionFor<Runtime, Block>;

substrate/client/consensus/beefy/src/justification.rs

@@ -20,7 +20,7 @@ use crate::keystore::BeefyKeystore;
 use codec::{DecodeAll, Encode};
 use sp_consensus::Error as ConsensusError;
 use sp_consensus_beefy::{
-	crypto::{AuthorityId, Signature},
+	ecdsa_crypto::{AuthorityId, Signature},
 	ValidatorSet, ValidatorSetId, VersionedFinalityProof,
 };
 use sp_runtime::traits::{Block as BlockT, NumberFor};

substrate/client/consensus/beefy/src/keystore.rs

@@ -16,15 +16,15 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-use sp_application_crypto::RuntimeAppPublic;
+use sp_application_crypto::{key_types::BEEFY as BEEFY_KEY_TYPE, RuntimeAppPublic};
 use sp_core::keccak_256;
 use sp_keystore::KeystorePtr;
 
 use log::warn;
 
 use sp_consensus_beefy::{
-	crypto::{Public, Signature},
-	BeefyAuthorityId, KEY_TYPE,
+	ecdsa_crypto::{Public, Signature},
+	BeefyAuthorityId,
 };
 
 use crate::{error, LOG_TARGET};
@@ -50,7 +50,7 @@ impl BeefyKeystore {
 		// we do check for multiple private keys as a key store sanity check.
 		let public: Vec<Public> = keys
 			.iter()
-			.filter(|k| store.has_keys(&[(k.to_raw_vec(), KEY_TYPE)]))
+			.filter(|k| store.has_keys(&[(k.to_raw_vec(), BEEFY_KEY_TYPE)]))
 			.cloned()
 			.collect();
 
@@ -78,7 +78,7 @@ impl BeefyKeystore {
 		let public = public.as_ref();
 
 		let sig = store
-			.ecdsa_sign_prehashed(KEY_TYPE, public, &msg)
+			.ecdsa_sign_prehashed(BEEFY_KEY_TYPE, public, &msg)
 			.map_err(|e| error::Error::Keystore(e.to_string()))?
 			.ok_or_else(|| error::Error::Signature("ecdsa_sign_prehashed() failed".to_string()))?;
 
@@ -96,7 +96,7 @@ impl BeefyKeystore {
 		let store = self.0.clone().ok_or_else(|| error::Error::Keystore("no Keystore".into()))?;
 
 		let pk: Vec<Public> =
-			store.ecdsa_public_keys(KEY_TYPE).drain(..).map(Public::from).collect();
+			store.ecdsa_public_keys(BEEFY_KEY_TYPE).drain(..).map(Public::from).collect();
 
 		Ok(pk)
 	}
@@ -117,7 +117,7 @@ impl From<Option<KeystorePtr>> for BeefyKeystore {
 
 #[cfg(test)]
 pub mod tests {
-	use sp_consensus_beefy::{crypto, Keyring};
+	use sp_consensus_beefy::{ecdsa_crypto, Keyring};
 	use sp_core::{ecdsa, Pair};
 	use sp_keystore::testing::MemoryKeystore;
 
@@ -156,35 +156,51 @@ pub mod tests {
 
 	#[test]
 	fn pair_works() {
-		let want = crypto::Pair::from_string("//Alice", None).expect("Pair failed").to_raw_vec();
+		let want = ecdsa_crypto::Pair::from_string("//Alice", None)
+			.expect("Pair failed")
+			.to_raw_vec();
 		let got = Keyring::Alice.pair().to_raw_vec();
 		assert_eq!(want, got);
 
-		let want = crypto::Pair::from_string("//Bob", None).expect("Pair failed").to_raw_vec();
+		let want = ecdsa_crypto::Pair::from_string("//Bob", None)
+			.expect("Pair failed")
+			.to_raw_vec();
 		let got = Keyring::Bob.pair().to_raw_vec();
 		assert_eq!(want, got);
 
-		let want = crypto::Pair::from_string("//Charlie", None).expect("Pair failed").to_raw_vec();
+		let want = ecdsa_crypto::Pair::from_string("//Charlie", None)
+			.expect("Pair failed")
+			.to_raw_vec();
 		let got = Keyring::Charlie.pair().to_raw_vec();
 		assert_eq!(want, got);
 
-		let want = crypto::Pair::from_string("//Dave", None).expect("Pair failed").to_raw_vec();
+		let want = ecdsa_crypto::Pair::from_string("//Dave", None)
+			.expect("Pair failed")
+			.to_raw_vec();
 		let got = Keyring::Dave.pair().to_raw_vec();
 		assert_eq!(want, got);
 
-		let want = crypto::Pair::from_string("//Eve", None).expect("Pair failed").to_raw_vec();
+		let want = ecdsa_crypto::Pair::from_string("//Eve", None)
+			.expect("Pair failed")
+			.to_raw_vec();
 		let got = Keyring::Eve.pair().to_raw_vec();
 		assert_eq!(want, got);
 
-		let want = crypto::Pair::from_string("//Ferdie", None).expect("Pair failed").to_raw_vec();
+		let want = ecdsa_crypto::Pair::from_string("//Ferdie", None)
+			.expect("Pair failed")
+			.to_raw_vec();
 		let got = Keyring::Ferdie.pair().to_raw_vec();
 		assert_eq!(want, got);
 
-		let want = crypto::Pair::from_string("//One", None).expect("Pair failed").to_raw_vec();
+		let want = ecdsa_crypto::Pair::from_string("//One", None)
+			.expect("Pair failed")
+			.to_raw_vec();
 		let got = Keyring::One.pair().to_raw_vec();
 		assert_eq!(want, got);
 
-		let want = crypto::Pair::from_string("//Two", None).expect("Pair failed").to_raw_vec();
+		let want = ecdsa_crypto::Pair::from_string("//Two", None)
+			.expect("Pair failed")
+			.to_raw_vec();
 		let got = Keyring::Two.pair().to_raw_vec();
 		assert_eq!(want, got);
 	}
@@ -193,8 +209,8 @@ pub mod tests {
 	fn authority_id_works() {
 		let store = keystore();
 
-		let alice: crypto::Public = store
-			.ecdsa_generate_new(KEY_TYPE, Some(&Keyring::Alice.to_seed()))
+		let alice: ecdsa_crypto::Public = store
+			.ecdsa_generate_new(BEEFY_KEY_TYPE, Some(&Keyring::Alice.to_seed()))
 			.ok()
 			.unwrap()
 			.into();
@@ -219,8 +235,8 @@ pub mod tests {
 	fn sign_works() {
 		let store = keystore();
 
-		let alice: crypto::Public = store
-			.ecdsa_generate_new(KEY_TYPE, Some(&Keyring::Alice.to_seed()))
+		let alice: ecdsa_crypto::Public = store
+			.ecdsa_generate_new(BEEFY_KEY_TYPE, Some(&Keyring::Alice.to_seed()))
 			.ok()
 			.unwrap()
 			.into();
@@ -239,7 +255,10 @@ pub mod tests {
 	fn sign_error() {
 		let store = keystore();
 
-		store.ecdsa_generate_new(KEY_TYPE, Some(&Keyring::Bob.to_seed())).ok().unwrap();
+		store
+			.ecdsa_generate_new(BEEFY_KEY_TYPE, Some(&Keyring::Bob.to_seed()))
+			.ok()
+			.unwrap();
 
 		let store: BeefyKeystore = Some(store).into();
 
@@ -268,8 +287,8 @@ pub mod tests {
 	fn verify_works() {
 		let store = keystore();
 
-		let alice: crypto::Public = store
-			.ecdsa_generate_new(KEY_TYPE, Some(&Keyring::Alice.to_seed()))
+		let alice: ecdsa_crypto::Public = store
+			.ecdsa_generate_new(BEEFY_KEY_TYPE, Some(&Keyring::Alice.to_seed()))
 			.ok()
 			.unwrap()
 			.into();
@@ -305,11 +324,11 @@ pub mod tests {
 		let _ = add_key(TEST_TYPE, None);
 
 		// BEEFY keys
-		let _ = add_key(KEY_TYPE, Some(Keyring::Dave.to_seed().as_str()));
-		let _ = add_key(KEY_TYPE, Some(Keyring::Eve.to_seed().as_str()));
+		let _ = add_key(BEEFY_KEY_TYPE, Some(Keyring::Dave.to_seed().as_str()));
+		let _ = add_key(BEEFY_KEY_TYPE, Some(Keyring::Eve.to_seed().as_str()));
 
-		let key1: crypto::Public = add_key(KEY_TYPE, None).into();
-		let key2: crypto::Public = add_key(KEY_TYPE, None).into();
+		let key1: ecdsa_crypto::Public = add_key(BEEFY_KEY_TYPE, None).into();
+		let key2: ecdsa_crypto::Public = add_key(BEEFY_KEY_TYPE, None).into();
 
 		let store: BeefyKeystore = Some(store).into();
 

substrate/client/consensus/beefy/src/lib.rs

@@ -46,7 +46,8 @@ use sp_blockchain::{
 };
 use sp_consensus::{Error as ConsensusError, SyncOracle};
 use sp_consensus_beefy::{
-	crypto::AuthorityId, BeefyApi, MmrRootHash, PayloadProvider, ValidatorSet, BEEFY_ENGINE_ID,
+	ecdsa_crypto::AuthorityId, BeefyApi, MmrRootHash, PayloadProvider, ValidatorSet,
+	BEEFY_ENGINE_ID,
 };
 use sp_keystore::KeystorePtr;
 use sp_mmr_primitives::MmrApi;
@@ -142,7 +143,7 @@ where
 		+ Send
 		+ Sync,
 	RuntimeApi: ProvideRuntimeApi<B> + Send + Sync,
-	RuntimeApi::Api: BeefyApi<B>,
+	RuntimeApi::Api: BeefyApi<B, AuthorityId>,
 {
 	// Voter -> RPC links
 	let (to_rpc_justif_sender, from_voter_justif_stream) =
@@ -224,7 +225,7 @@ pub async fn start_beefy_gadget<B, BE, C, N, P, R, S>(
 	C: Client<B, BE> + BlockBackend<B>,
 	P: PayloadProvider<B>,
 	R: ProvideRuntimeApi<B>,
-	R::Api: BeefyApi<B> + MmrApi<B, MmrRootHash, NumberFor<B>>,
+	R::Api: BeefyApi<B, AuthorityId> + MmrApi<B, MmrRootHash, NumberFor<B>>,
 	N: GossipNetwork<B> + NetworkRequest + Send + Sync + 'static,
 	S: GossipSyncing<B> + SyncOracle + 'static,
 {
@@ -339,7 +340,7 @@ where
 	B: Block,
 	BE: Backend<B>,
 	R: ProvideRuntimeApi<B>,
-	R::Api: BeefyApi<B>,
+	R::Api: BeefyApi<B, AuthorityId>,
 {
 	// Initialize voter state from AUX DB if compatible.
 	crate::aux_schema::load_persistent(backend)?
@@ -374,7 +375,7 @@ where
 	B: Block,
 	BE: Backend<B>,
 	R: ProvideRuntimeApi<B>,
-	R::Api: BeefyApi<B>,
+	R::Api: BeefyApi<B, AuthorityId>,
 {
 	let beefy_genesis = runtime
 		.runtime_api()
@@ -478,7 +479,7 @@ async fn wait_for_runtime_pallet<B, R>(
 where
 	B: Block,
 	R: ProvideRuntimeApi<B>,
-	R::Api: BeefyApi<B>,
+	R::Api: BeefyApi<B, AuthorityId>,
 {
 	info!(target: LOG_TARGET, "🥩 BEEFY gadget waiting for BEEFY pallet to become available...");
 	loop {
@@ -518,7 +519,7 @@ fn expect_validator_set<B, R>(
 where
 	B: Block,
 	R: ProvideRuntimeApi<B>,
-	R::Api: BeefyApi<B>,
+	R::Api: BeefyApi<B, AuthorityId>,
 {
 	runtime
 		.runtime_api()

substrate/client/consensus/beefy/src/round.rs

@@ -21,7 +21,7 @@ use crate::LOG_TARGET;
 use codec::{Decode, Encode};
 use log::debug;
 use sp_consensus_beefy::{
-	crypto::{AuthorityId, Signature},
+	ecdsa_crypto::{AuthorityId, Signature},
 	Commitment, EquivocationProof, SignedCommitment, ValidatorSet, ValidatorSetId, VoteMessage,
 };
 use sp_runtime::traits::{Block, NumberFor};

substrate/client/consensus/beefy/src/tests.rs

@@ -48,14 +48,15 @@ use sc_network_test::{
 use sc_utils::notification::NotificationReceiver;
 use serde::{Deserialize, Serialize};
 use sp_api::{ApiRef, ProvideRuntimeApi};
+use sp_application_crypto::key_types::BEEFY as BEEFY_KEY_TYPE;
 use sp_consensus::BlockOrigin;
 use sp_consensus_beefy::{
-	crypto::{AuthorityId, Signature},
+	ecdsa_crypto::{AuthorityId, Signature},
 	known_payloads,
 	mmr::{find_mmr_root_digest, MmrRootProvider},
 	BeefyApi, Commitment, ConsensusLog, EquivocationProof, Keyring as BeefyKeyring, MmrRootHash,
 	OpaqueKeyOwnershipProof, Payload, SignedCommitment, ValidatorSet, ValidatorSetId,
-	VersionedFinalityProof, VoteMessage, BEEFY_ENGINE_ID, KEY_TYPE as BeefyKeyType,
+	VersionedFinalityProof, VoteMessage, BEEFY_ENGINE_ID,
 };
 use sp_core::H256;
 use sp_keystore::{testing::MemoryKeystore, Keystore, KeystorePtr};
@@ -293,7 +294,7 @@ impl ProvideRuntimeApi<Block> for TestApi {
 	}
 }
 sp_api::mock_impl_runtime_apis! {
-	impl BeefyApi<Block> for RuntimeApi {
+	impl BeefyApi<Block, AuthorityId> for RuntimeApi {
 		fn beefy_genesis() -> Option<NumberFor<Block>> {
 			Some(self.inner.beefy_genesis)
 		}
@@ -352,7 +353,7 @@ pub(crate) fn make_beefy_ids(keys: &[BeefyKeyring]) -> Vec<AuthorityId> {
 pub(crate) fn create_beefy_keystore(authority: BeefyKeyring) -> KeystorePtr {
 	let keystore = MemoryKeystore::new();
 	keystore
-		.ecdsa_generate_new(BeefyKeyType, Some(&authority.to_seed()))
+		.ecdsa_generate_new(BEEFY_KEY_TYPE, Some(&authority.to_seed()))
 		.expect("Creates authority key");
 	keystore.into()
 }
@@ -386,7 +387,7 @@ fn initialize_beefy<API>(
 ) -> impl Future<Output = ()>
 where
 	API: ProvideRuntimeApi<Block> + Sync + Send,
-	API::Api: BeefyApi<Block> + MmrApi<Block, MmrRootHash, NumberFor<Block>>,
+	API::Api: BeefyApi<Block, AuthorityId> + MmrApi<Block, MmrRootHash, NumberFor<Block>>,
 {
 	let tasks = FuturesUnordered::new();
 

substrate/client/consensus/beefy/src/worker.rs

@@ -41,7 +41,7 @@ use sp_arithmetic::traits::{AtLeast32Bit, Saturating};
 use sp_consensus::SyncOracle;
 use sp_consensus_beefy::{
 	check_equivocation_proof,
-	crypto::{AuthorityId, Signature},
+	ecdsa_crypto::{AuthorityId, Signature},
 	BeefyApi, Commitment, ConsensusLog, EquivocationProof, PayloadProvider, ValidatorSet,
 	VersionedFinalityProof, VoteMessage, BEEFY_ENGINE_ID,
 };
@@ -348,7 +348,7 @@ where
 	P: PayloadProvider<B>,
 	S: SyncOracle,
 	R: ProvideRuntimeApi<B>,
-	R::Api: BeefyApi<B>,
+	R::Api: BeefyApi<B, AuthorityId>,
 {
 	fn best_grandpa_block(&self) -> NumberFor<B> {
 		*self.persisted_state.voting_oracle.best_grandpa_block_header.number()