Offchain signing (#5182)

* New approach to offchain signing.

* Use in im-online

* Rewrite to use Account<T>

* DRY signing.

* Implement send_raw_unsigned_transaction

* WiP

* Expunge LocalCall

* Expunge LocalCall

* Fix compilation.

* Solve call.

* Make it compile again.

* Finalize implementation.

* Change CreateTransaction

* Clear CreateTransaction.

* Add price payload

* Send raw transaction

* Submit signed payload / unsigned transaction (WIP)

* Supertrait requirements on T::Signature

* Validate signature of payload on an unsigned transaction

* Fix encoding - part 1

* Make it compile.

* Fix compilation of unsigned validator.

* Pass price payload to the transaction

* Make block number part of the signed payload

* Send signed transaction

* Implement all_accounts, any_account

* Fix formatting

* Implement submit_transaction

* Submit signed transaction (ForAll, ForAny)

* Fix formatting

* Implement CreateSignedTransaction

* Move sign and verify to AppCrypto

* Sign transaction

* Call `use_encoded`

* Remove SubmitAndSignTransaction

* Implement runtime using new SigningTypes

* Adapt offchain example to changes

* Fix im-online pallet

* Quick fix: rename AuthorityId2

* Fix offchain example tests

* Add a comment on why keystore is required in unsigned transaction test

* Use UintAuthorityId instead of u64

* WIP

* Remove IdentifyAccount from UintAuthorityId

* Implement PublicWrapper type

* Fix im-online tests

* Fix runtime test

* Bump spec version

* Fix executor tests

* Rename ImOnlineAuthId -> ImOnlineAuthorityId and formatting

* Fix merge

* Documentation

* Revert u64 -> UintAuthorityId conversion

* Fix string errors

* Document public members in offchain module

* Introduce SubmitTransaction

* Update pallets to use SubmitTransaction

* WIP

* Use SubmitTransaction in offchain

* Use `submit_unsigned_transaction`

* Fix tests

* Update docs

* Remove SigningTypes requirement from `SendTransactionTypes`

* Fix tests

* Update frame/system/src/offchain.rs

Co-Authored-By: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Update frame/system/src/offchain.rs

Co-Authored-By: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Update frame/example-offchain-worker/src/tests.rs

Co-Authored-By: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Update frame/system/src/offchain.rs

Co-Authored-By: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Update frame/system/src/offchain.rs

Co-Authored-By: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Remove leftover from previous iterations

* Change enum to struct

* Remove public

* Move mock to node/executor/tests

* Cleanup test-helpers

* Make `application-crypto` `std` feature internal

The macros should not generate code that requires that the calling crate
has a feature with the name `std` defined.

* Revert cargo lock update

* Use TestAuthorityId from common

* Restore members of account to public

* Tidy up imports

* Fix benchmarking pallet

* Add tests demonstrating ForAll, ForAny on signer

* Move definition of AppCrypto

in example-offchain-worker
from tests to mod::crypto

* Cleanup stray comment

* Fix ValidTransaction

* Re-fix CreateSignedTransaction

* Address PR feedback

* Add can_sign method to signer

* Propagate error

* Improve documentation

* Fix vec! macro not available

* Document SendTransactiontypes

* Add some docs.

* Split signing examples

* Add tests for signing examples

* WIP can_sign - PR feedback

* WIP

* Split for_any / for_all into different calls

* Verify payload and signature in test

* Fix can_sign implementation

* Fix impl_version

* Import Box from sp_std

* Create issues for TODOs

* Ignore doctest.

* Add test directly to system. Adjust UintTypes.

* Add some tests to account filtering.

* Remove code samples and point to example offchain worker

* Fix doc links

* Fix im-online tests using signatures.

Co-authored-by: Tomasz Drwięga <tomasz@parity.io>
Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
Co-authored-by: Bastian Köcher <git@kchr.de>

substrate/frame/im-online/src/lib.rs

@@ -98,7 +98,10 @@ use frame_support::{
 	weights::{SimpleDispatchInfo, MINIMUM_WEIGHT},
 };
 use frame_system::{self as system, ensure_none};
-use frame_system::offchain::SubmitUnsignedTransaction;
+use frame_system::offchain::{
+	SendTransactionTypes,
+	SubmitTransaction,
+};
 
 pub mod sr25519 {
 	mod app_sr25519 {
@@ -221,19 +224,13 @@ pub struct Heartbeat<BlockNumber>
 	pub authority_index: AuthIndex,
 }
 
-pub trait Trait: frame_system::Trait + pallet_session::historical::Trait {
+pub trait Trait: SendTransactionTypes<Call<Self>> + pallet_session::historical::Trait {
 	/// The identifier type for an authority.
 	type AuthorityId: Member + Parameter + RuntimeAppPublic + Default + Ord;
 
 	/// The overarching event type.
 	type Event: From<Event<Self>> + Into<<Self as frame_system::Trait>::Event>;
 
-	/// A dispatchable call type.
-	type Call: From<Call<Self>>;
-
-	/// A transaction submitter.
-	type SubmitTransaction: SubmitUnsignedTransaction<Self, <Self as Trait>::Call>;
-
 	/// An expected duration of the session.
 	///
 	/// This parameter is used to determine the longevity of `heartbeat` transaction
@@ -444,6 +441,7 @@ impl<T: Trait> Module<T> {
 		}
 
 		let session_index = <pallet_session::Module<T>>::current_index();
+
 		Ok(Self::local_authority_keys()
 			.map(move |(authority_index, key)|
 				Self::send_single_heartbeat(authority_index, key, session_index, block_number)
@@ -467,7 +465,9 @@ impl<T: Trait> Module<T> {
 				session_index,
 				authority_index,
 			};
+
 			let signature = key.sign(&heartbeat_data.encode()).ok_or(OffchainErr::FailedSigning)?;
+
 			Ok(Call::heartbeat(heartbeat_data, signature))
 		};
 
@@ -492,7 +492,7 @@ impl<T: Trait> Module<T> {
 					call,
 				);
 
-				T::SubmitTransaction::submit_unsigned(call)
+				SubmitTransaction::<T, Call<T>>::submit_unsigned_transaction(call.into())
 					.map_err(|_| OffchainErr::SubmitTransaction)?;
 
 				Ok(())
@@ -501,9 +501,18 @@ impl<T: Trait> Module<T> {
 	}
 
 	fn local_authority_keys() -> impl Iterator<Item=(u32, T::AuthorityId)> {
-		// we run only when a local authority key is configured
+		// on-chain storage
+		//
+		// At index `idx`:
+		// 1. A (ImOnline) public key to be used by a validator at index `idx` to send im-online
+		//          heartbeats.
 		let authorities = Keys::<T>::get();
+
+		// local keystore
+		//
+		// All `ImOnline` public (+private) keys currently in the local keystore.
 		let mut local_keys = T::AuthorityId::all();
+
 		local_keys.sort();
 
 		authorities.into_iter()
@@ -565,6 +574,11 @@ impl<T: Trait> Module<T> {
 			Keys::<T>::put(keys);
 		}
 	}
+
+	#[cfg(test)]
+	fn set_keys(keys: Vec<T::AuthorityId>) {
+		Keys::<T>::put(&keys)
+	}
 }
 
 impl<T: Trait> sp_runtime::BoundToRuntimeAppPublic for Module<T> {

substrate/frame/im-online/src/mock.rs

@@ -27,7 +27,6 @@ use sp_runtime::testing::{Header, UintAuthorityId, TestXt};
 use sp_runtime::traits::{IdentityLookup, BlakeTwo256, ConvertInto};
 use sp_core::H256;
 use frame_support::{impl_outer_origin, impl_outer_dispatch, parameter_types, weights::Weight};
-
 use frame_system as system;
 impl_outer_origin!{
 	pub enum Origin for Runtime {}
@@ -40,7 +39,11 @@ impl_outer_dispatch! {
 }
 
 thread_local! {
-	pub static VALIDATORS: RefCell<Option<Vec<u64>>> = RefCell::new(Some(vec![1, 2, 3]));
+	pub static VALIDATORS: RefCell<Option<Vec<u64>>> = RefCell::new(Some(vec![
+		1,
+		2,
+		3,
+	]));
 }
 
 pub struct TestSessionManager;
@@ -68,7 +71,6 @@ impl pallet_session::historical::SessionManager<u64, u64> for TestSessionManager
 
 /// An extrinsic type used for tests.
 pub type Extrinsic = TestXt<Call, ()>;
-type SubmitTransaction = frame_system::offchain::TransactionSubmitter<(), Call, Extrinsic>;
 type IdentificationTuple = (u64, u64);
 type Offence = crate::UnresponsivenessOffence<IdentificationTuple>;
 
@@ -90,7 +92,6 @@ pub fn new_test_ext() -> sp_io::TestExternalities {
 	t.into()
 }
 
-
 #[derive(Clone, PartialEq, Eq, Debug)]
 pub struct Runtime;
 
@@ -168,13 +169,18 @@ parameter_types! {
 impl Trait for Runtime {
 	type AuthorityId = UintAuthorityId;
 	type Event = ();
-	type Call = Call;
-	type SubmitTransaction = SubmitTransaction;
 	type ReportUnresponsiveness = OffenceHandler;
 	type SessionDuration = Period;
 	type UnsignedPriority = UnsignedPriority;
 }
 
+impl<LocalCall> frame_system::offchain::SendTransactionTypes<LocalCall> for Runtime where
+	Call: From<LocalCall>,
+{
+	type OverarchingCall = Call;
+	type Extrinsic = Extrinsic;
+}
+
 /// Im Online module.
 pub type ImOnline = Module<Runtime>;
 pub type System = frame_system::Module<Runtime>;
@@ -184,5 +190,7 @@ pub fn advance_session() {
 	let now = System::block_number().max(1);
 	System::set_block_number(now + 1);
 	Session::rotate_session();
+	let keys = Session::validators().into_iter().map(UintAuthorityId).collect();
+	ImOnline::set_keys(keys);
 	assert_eq!(Session::current_index(), (now / Period::get()) as u32);
 }

substrate/frame/im-online/src/tests.rs

@@ -61,15 +61,15 @@ fn should_report_offline_validators() {
 		let block = 1;
 		System::set_block_number(block);
 		// buffer new validators
-		Session::rotate_session();
+		advance_session();
 		// enact the change and buffer another one
 		let validators = vec![1, 2, 3, 4, 5, 6];
 		VALIDATORS.with(|l| *l.borrow_mut() = Some(validators.clone()));
-		Session::rotate_session();
+		advance_session();
 
 		// when
 		// we end current session and start the next one
-		Session::rotate_session();
+		advance_session();
 
 		// then
 		let offences = OFFENCES.with(|l| l.replace(vec![]));
@@ -89,7 +89,7 @@ fn should_report_offline_validators() {
 		for (idx, v) in validators.into_iter().take(4).enumerate() {
 			let _ = heartbeat(block, 3, idx as u32, v.into()).unwrap();
 		}
-		Session::rotate_session();
+		advance_session();
 
 		// then
 		let offences = OFFENCES.with(|l| l.replace(vec![]));
@@ -174,7 +174,7 @@ fn late_heartbeat_should_fail() {
 	new_test_ext().execute_with(|| {
 		advance_session();
 		// given
-		VALIDATORS.with(|l| *l.borrow_mut() = Some(vec![1, 2, 4, 4, 5, 6]));
+		VALIDATORS.with(|l| *l.borrow_mut() = Some(vec![1, 2, 3, 4, 5, 6]));
 		assert_eq!(Session::validators(), Vec::<u64>::new());
 		// enact the change and buffer another one
 		advance_session();
@@ -315,7 +315,7 @@ fn should_not_send_a_report_if_already_online() {
 		ImOnline::note_uncle(3, 0);
 
 		// when
-		UintAuthorityId::set_all_keys(vec![0]); // all authorities use pallet_session key 0
+		UintAuthorityId::set_all_keys(vec![1, 2, 3]);
 		// we expect error, since the authority is already online.
 		let mut res = ImOnline::send_heartbeats(4).unwrap();
 		assert_eq!(res.next().unwrap().unwrap(), ());