Support for multiple signature scheme for BEEFY primitves (#14373)

* Merged BEEFY primitives with generic signature and keyset commitment support from old pull to current code

* - Add bls-experimental feature to application-crypto and beefy primitives
- Fix remaining crypto -> ecdsa_crypto
- code build but not tests

* Make beefy primitive tests compile

* move bls related beefy primitives code and test behind bls-experimental flag

* Make BEEFY clients complies with BEEFY API depending on AuthorityId

* - Rename `BeefyAuthoritySet.root` → `BeefyAuthoritySet.keyset_commitment`.
- Remove apk proof keyset_commitment from `BeefyAuthoritySet`.
- Fix failing signed commitment and signature to witness test.
- Make client compatible with BeefyAPI generic on AuthorityId.
- `crypto` → `ecdsa_crypto` in BEEFY client and frame.

* Commit Cargo lock remove ark-serialize from BEEFY primitives

* Use Codec instead of Encode + Decode in primitives/consensus/beefy/src/lib.rs

Co-authored-by: Davide Galassi <davxy@datawok.net>

* - Make `BeefyApi` generic over Signature type.
- Make new `BeeyApi` functinos also generic over AuthorityId and Signature

* Unmake BeefyAPI generic over Signature. Recover Signature type from AuthId.

* - dont use hex or hex-literal use array-bytes instead in beefy primitives and bls crypto.
- CamelCase ECDSA and BLS everywhere.

* Move the definition of BEEFY key type from `primitives/beefy` to `crypto.rs` according to new convention.

* - Add bls377_generate_new to `sp-io` and `application_crypto::bls`.
- Add `bls-experimental` to `sp-io`

Does not compile because PassByCodec can not derive PassBy using customly implemented PassByIner.

* Implement PassBy for `bls::Public` manually

* fix Beefy `KEY_TYPE` in `frame/beefy` tests to come from `sp-core::key_types` enum

* specify both generic for `hex2array_unchecked` in `sp-core/bls.rs`

* Rename `crypto`→`ecdsa_crypto` in `primitives/consensus/beefy/src/test_utils.rs` docs

* remove commented-out code in `primitives/consensus/beefy/src/commitment.rs`

Co-authored-by: Davide Galassi <davxy@datawok.net>

* Fix inconsistency in panic message in  `primitives/io/src/lib.rs`

Co-authored-by: Davide Galassi <davxy@datawok.net>

* Remove redundant feature activation in `primitives/io/Cargo.toml`

Co-authored-by: Davide Galassi <davxy@datawok.net>

* - make `w3f-bls` a dev-dependancy only for beefy primitives.

- clean up comments.

Co-authored-by: Davide Galassi <davxy@datawok.net>

* export BEEFY KEY_TYPE from primitives/consensus/beefy
make `frame/consensus/beefy` in dependent of sp_crypto_app
use consistent naming in the beefy primitive tests.

* - implement `BeefyAuthorityId` for `bls_crypto::AuthorityId`.
- implement `bls_verify_works` test for BEEFY `bls_crypto`.

* Remove BEEFY `ecdsa_n_bls_crypto` for now for later re-introduction

* Make commitment and witness BEEFY tests not use Keystore.

* put `bls_beefy_verify_works` test under `bls-experimental` flag.

* bump up Runtime `BeefyAPI` to version 3 due to introducing generic AuthorityId.

* reuse code and encapsulate w3f-bls backend in sp-core as most as possible

Co-authored-by: Davide Galassi <davxy@datawok.net>

* Make comments in primities BEEFY `commitment.rs` and `witness.rs``tests convention conforming

* Use master dep versions

* Trivial change. Mostly to trigger CI

* Apply suggestions from code review

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

* Fix Cargo.toml

* Trigger CI with cumulus companion

* Trigger CI after polkadot companion change

---------

Co-authored-by: Davide Galassi <davxy@datawok.net>
Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

substrate/frame/beefy-mmr/src/lib.rs

@@ -73,8 +73,8 @@ where
 
 /// Convert BEEFY secp256k1 public keys into Ethereum addresses
 pub struct BeefyEcdsaToEthereum;
-impl Convert<sp_consensus_beefy::crypto::AuthorityId, Vec<u8>> for BeefyEcdsaToEthereum {
-	fn convert(beefy_id: sp_consensus_beefy::crypto::AuthorityId) -> Vec<u8> {
+impl Convert<sp_consensus_beefy::ecdsa_crypto::AuthorityId, Vec<u8>> for BeefyEcdsaToEthereum {
+	fn convert(beefy_id: sp_consensus_beefy::ecdsa_crypto::AuthorityId) -> Vec<u8> {
 		sp_core::ecdsa::Public::from(beefy_id)
 			.to_eth_address()
 			.map(|v| v.to_vec())
@@ -200,11 +200,12 @@ impl<T: Config> Pallet<T> {
 			.map(T::BeefyAuthorityToMerkleLeaf::convert)
 			.collect::<Vec<_>>();
 		let len = beefy_addresses.len() as u32;
-		let root = binary_merkle_tree::merkle_root::<<T as pallet_mmr::Config>::Hashing, _>(
-			beefy_addresses,
-		)
+		let keyset_commitment = binary_merkle_tree::merkle_root::<
+			<T as pallet_mmr::Config>::Hashing,
+			_,
+		>(beefy_addresses)
 		.into();
-		BeefyAuthoritySet { id, len, root }
+		BeefyAuthoritySet { id, len, keyset_commitment }
 	}
 }
 

substrate/frame/beefy-mmr/src/mock.rs

@@ -36,7 +36,7 @@ use sp_runtime::{
 use crate as pallet_beefy_mmr;
 
 pub use sp_consensus_beefy::{
-	crypto::AuthorityId as BeefyId, mmr::BeefyDataProvider, ConsensusLog, BEEFY_ENGINE_ID,
+	ecdsa_crypto::AuthorityId as BeefyId, mmr::BeefyDataProvider, ConsensusLog, BEEFY_ENGINE_ID,
 };
 
 impl_opaque_keys! {

substrate/frame/beefy-mmr/src/tests.rs

@@ -119,7 +119,7 @@ fn should_contain_valid_leaf_data() {
 			beefy_next_authority_set: BeefyNextAuthoritySet {
 				id: 2,
 				len: 2,
-				root: array_bytes::hex_n_into_unchecked(
+				keyset_commitment: array_bytes::hex_n_into_unchecked(
 					"9c6b2c1b0d0b25a008e6c882cc7b415f309965c72ad2b944ac0931048ca31cd5"
 				)
 			},
@@ -144,7 +144,7 @@ fn should_contain_valid_leaf_data() {
 			beefy_next_authority_set: BeefyNextAuthoritySet {
 				id: 3,
 				len: 2,
-				root: array_bytes::hex_n_into_unchecked(
+				keyset_commitment: array_bytes::hex_n_into_unchecked(
 					"9c6b2c1b0d0b25a008e6c882cc7b415f309965c72ad2b944ac0931048ca31cd5"
 				)
 			},
@@ -167,12 +167,12 @@ fn should_update_authorities() {
 		let want = array_bytes::hex_n_into_unchecked::<_, H256, 32>(
 			"176e73f1bf656478b728e28dd1a7733c98621b8acf830bff585949763dca7a96",
 		);
-		assert_eq!(want, auth_set.root);
+		assert_eq!(want, auth_set.keyset_commitment);
 
 		// next authority set should have same validators but different id
 		assert_eq!(1, next_auth_set.id);
 		assert_eq!(auth_set.len, next_auth_set.len);
-		assert_eq!(auth_set.root, next_auth_set.root);
+		assert_eq!(auth_set.keyset_commitment, next_auth_set.keyset_commitment);
 
 		let announced_set = next_auth_set;
 		init_block(1);
@@ -188,7 +188,7 @@ fn should_update_authorities() {
 			"9c6b2c1b0d0b25a008e6c882cc7b415f309965c72ad2b944ac0931048ca31cd5",
 		);
 		assert_eq!(2, next_auth_set.len);
-		assert_eq!(want, next_auth_set.root);
+		assert_eq!(want, next_auth_set.keyset_commitment);
 
 		let announced_set = next_auth_set;
 		init_block(2);
@@ -204,6 +204,6 @@ fn should_update_authorities() {
 			"9c6b2c1b0d0b25a008e6c882cc7b415f309965c72ad2b944ac0931048ca31cd5",
 		);
 		assert_eq!(2, next_auth_set.len);
-		assert_eq!(want, next_auth_set.root);
+		assert_eq!(want, next_auth_set.keyset_commitment);
 	});
 }