Reorganising the repository - external renames and moves (#4074)

* Adding first rough ouline of the repository structure

* Remove old CI stuff

* add title

* formatting fixes

* move node-exits job's script to scripts dir

* Move docs into subdir

* move to bin

* move maintainence scripts, configs and helpers into its own dir

* add .local to ignore

* move core->client

* start up 'test' area

* move test client

* move test runtime

* make test move compile

* Add dependencies rule enforcement.

* Fix indexing.

* Update docs to reflect latest changes

* Moving /srml->/paint

* update docs

* move client/sr-* -> primitives/

* clean old readme

* remove old broken code in rhd

* update lock

* Step 1.

* starting to untangle client

* Fix after merge.

* start splitting out client interfaces

* move children and blockchain interfaces

* Move trie and state-machine to primitives.

* Fix WASM builds.

* fixing broken imports

* more interface moves

* move backend and light to interfaces

* move CallExecutor

* move cli off client

* moving around more interfaces

* re-add consensus crates into the mix

* fix subkey path

* relieve client from executor

* starting to pull out client from grandpa

* move is_decendent_of out of client

* grandpa still depends on client directly

* lemme tests pass

* rename srml->paint

* Make it compile.

* rename interfaces->client-api

* Move keyring to primitives.

* fixup libp2p dep

* fix broken use

* allow dependency enforcement to fail

* move fork-tree

* Moving wasm-builder

* make env

* move build-script-utils

* fixup broken crate depdencies and names

* fix imports for authority discovery

* fix typo

* update cargo.lock

* fixing imports

* Fix paths and add missing crates

* re-add missing crates

substrate/primitives/application-crypto/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "substrate-application-crypto"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+description = "Provides facilities for generating application specific crypto wrapper types."
+
+[dependencies]
+primitives = { package = "substrate-primitives", path = "../core", default-features = false }
+codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] }
+serde = { version = "1.0.101", optional = true, features = ["derive"] }
+rstd = { package = "sr-std", path = "../sr-std", default-features = false }
+runtime-io = { package = "sr-io", path = "../../primitives/sr-io", default-features = false }
+
+[dev-dependencies]
+test-client = { package = "substrate-test-runtime-client", path = "../../test/utils/runtime/client" }
+sr-primitives = { path = "../../primitives/sr-primitives" }
+
+[features]
+default = [ "std" ]
+std = [ "full_crypto", "primitives/std", "codec/std", "serde", "rstd/std", "runtime-io/std" ]
+
+# This feature enables all crypto primitives for `no_std` builds like microcontrollers 
+# or Intel SGX. 
+# For the regular wasm runtime builds this should not be used.
+full_crypto = [
+    "primitives/full_crypto"
+]

substrate/primitives/application-crypto/src/ed25519.rs

@@ -0,0 +1,80 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Ed25519 crypto types.
+
+use crate::{RuntimePublic, KeyTypeId};
+
+use rstd::vec::Vec;
+
+pub use primitives::ed25519::*;
+
+mod app {
+	use primitives::testing::ED25519;
+	crate::app_crypto!(super, ED25519);
+
+	impl crate::traits::BoundToRuntimeAppPublic for Public {
+		type Public = Self;
+	}
+}
+
+pub use app::{Public as AppPublic, Signature as AppSignature};
+#[cfg(feature = "full_crypto")]
+pub use app::Pair as AppPair;
+
+impl RuntimePublic for Public {
+	type Signature = Signature;
+
+	fn all(key_type: KeyTypeId) -> crate::Vec<Self> {
+		runtime_io::crypto::ed25519_public_keys(key_type)
+	}
+
+	fn generate_pair(key_type: KeyTypeId, seed: Option<Vec<u8>>) -> Self {
+		runtime_io::crypto::ed25519_generate(key_type, seed)
+	}
+
+	fn sign<M: AsRef<[u8]>>(&self, key_type: KeyTypeId, msg: &M) -> Option<Self::Signature> {
+		runtime_io::crypto::ed25519_sign(key_type, self, msg.as_ref())
+	}
+
+	fn verify<M: AsRef<[u8]>>(&self, msg: &M, signature: &Self::Signature) -> bool {
+		runtime_io::crypto::ed25519_verify(&signature, msg.as_ref(), self)
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use sr_primitives::{generic::BlockId, traits::ProvideRuntimeApi};
+	use primitives::{testing::{KeyStore, ED25519}, crypto::Pair};
+	use test_client::{
+		TestClientBuilder, DefaultTestClientBuilderExt, TestClientBuilderExt,
+		runtime::{TestAPI, app_crypto::ed25519::{AppPair, AppPublic}},
+	};
+
+	#[test]
+	fn ed25519_works_in_runtime() {
+		let keystore = KeyStore::new();
+		let test_client = TestClientBuilder::new().set_keystore(keystore.clone()).build();
+		let (signature, public) = test_client.runtime_api()
+			.test_ed25519_crypto(&BlockId::Number(0))
+			.expect("Tests `ed25519` crypto.");
+
+		let key_pair = keystore.read().ed25519_key_pair(ED25519, &public.as_ref())
+			.expect("There should be at a `ed25519` key in the keystore for the given public key.");
+
+		assert!(AppPair::verify(&signature, "ed25519", &AppPublic::from(key_pair.public())));
+	}
+}

substrate/primitives/application-crypto/src/lib.rs

@@ -0,0 +1,434 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Traits and macros for constructing application specific strongly typed crypto wrappers.
+
+#![warn(missing_docs)]
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+#[doc(hidden)]
+pub use primitives::{self, crypto::{CryptoType, Public, Derive, IsWrappedBy, Wraps}, RuntimeDebug};
+#[doc(hidden)]
+#[cfg(feature = "full_crypto")]
+pub use primitives::crypto::{SecretStringError, DeriveJunction, Ss58Codec, Pair};
+pub use primitives::{crypto::{KeyTypeId, key_types}};
+
+#[doc(hidden)]
+pub use codec;
+#[doc(hidden)]
+#[cfg(feature = "std")]
+pub use serde;
+#[doc(hidden)]
+pub use rstd::{ops::Deref, vec::Vec};
+
+pub mod ed25519;
+pub mod sr25519;
+mod traits;
+
+pub use traits::*;
+
+/// Declares Public, Pair, Signature types which are functionally equivalent to `$pair`, but are new
+/// Application-specific types whose identifier is `$key_type`.
+///
+/// ```rust
+///# use substrate_application_crypto::{app_crypto, wrap, ed25519, KeyTypeId};
+/// // Declare a new set of crypto types using Ed25519 logic that identifies as `KeyTypeId`
+/// // of value `b"fuba"`.
+/// app_crypto!(ed25519, KeyTypeId(*b"_uba"));
+/// ```
+#[cfg(feature = "full_crypto")]
+#[macro_export]
+macro_rules! app_crypto {
+	($module:ident, $key_type:expr) => {
+		$crate::app_crypto_public_full_crypto!($module::Public, $key_type);
+		$crate::app_crypto_public_common!($module::Public, $module::Signature, $key_type);
+		$crate::app_crypto_signature_full_crypto!($module::Signature, $key_type);
+		$crate::app_crypto_signature_common!($module::Signature, $key_type);
+		$crate::app_crypto_pair!($module::Pair, $key_type);
+	};
+}
+
+/// Declares Public, Pair, Signature types which are functionally equivalent to `$pair`, but are new
+/// Application-specific types whose identifier is `$key_type`.
+///
+/// ```rust
+///# use substrate_application_crypto::{app_crypto, wrap, ed25519, KeyTypeId};
+/// // Declare a new set of crypto types using Ed25519 logic that identifies as `KeyTypeId`
+/// // of value `b"fuba"`.
+/// app_crypto!(ed25519, KeyTypeId(*b"_uba"));
+/// ```
+#[cfg(not(feature = "full_crypto"))]
+#[macro_export]
+macro_rules! app_crypto {
+	($module:ident, $key_type:expr) => {
+		$crate::app_crypto_public_not_full_crypto!($module::Public, $key_type);
+		$crate::app_crypto_public_common!($module::Public, $module::Signature, $key_type);
+		$crate::app_crypto_signature_not_full_crypto!($module::Signature, $key_type);
+		$crate::app_crypto_signature_common!($module::Signature, $key_type);
+	};
+}
+
+/// Declares Pair type which is functionally equivalent to `$pair`, but is new
+/// Application-specific type whose identifier is `$key_type`.
+#[macro_export]
+macro_rules! app_crypto_pair {
+	($pair:ty, $key_type:expr) => {
+		$crate::wrap!{
+			/// A generic `AppPublic` wrapper type over $pair crypto; this has no specific App.
+			#[derive(Clone)]
+			pub struct Pair($pair);
+		}
+
+		impl $crate::CryptoType for Pair {
+			type Pair = Pair;
+		}
+
+		impl $crate::Pair for Pair {
+			type Public = Public;
+			type Seed = <$pair as $crate::Pair>::Seed;
+			type Signature = Signature;
+			type DeriveError = <$pair as $crate::Pair>::DeriveError;
+			
+			#[cfg(feature = "std")]
+			fn generate_with_phrase(password: Option<&str>) -> (Self, String, Self::Seed) {
+				let r = <$pair>::generate_with_phrase(password);
+				(Self(r.0), r.1, r.2)
+			}
+			#[cfg(feature = "std")]
+			fn from_phrase(phrase: &str, password: Option<&str>)
+				-> Result<(Self, Self::Seed), $crate::SecretStringError>
+			{
+				<$pair>::from_phrase(phrase, password).map(|r| (Self(r.0), r.1))
+			}
+			fn derive<
+				Iter: Iterator<Item=$crate::DeriveJunction>
+			>(&self, path: Iter, seed: Option<Self::Seed>) -> Result<(Self, Option<Self::Seed>), Self::DeriveError> {
+				self.0.derive(path, seed).map(|x| (Self(x.0), x.1))
+			}
+			fn from_seed(seed: &Self::Seed) -> Self { Self(<$pair>::from_seed(seed)) }
+			fn from_seed_slice(seed: &[u8]) -> Result<Self, $crate::SecretStringError> {
+				<$pair>::from_seed_slice(seed).map(Self)
+			}
+			fn sign(&self, msg: &[u8]) -> Self::Signature {
+				Signature(self.0.sign(msg))
+			}
+			fn verify<M: AsRef<[u8]>>(
+				sig: &Self::Signature,
+				message: M,
+				pubkey: &Self::Public,
+			) -> bool {
+				<$pair>::verify(&sig.0, message, pubkey.as_ref())
+			}
+			fn verify_weak<P: AsRef<[u8]>, M: AsRef<[u8]>>(
+				sig: &[u8],
+				message: M,
+				pubkey: P,
+			) -> bool {
+				<$pair>::verify_weak(sig, message, pubkey)
+			}
+			fn public(&self) -> Self::Public { Public(self.0.public()) }
+			fn to_raw_vec(&self) -> Vec<u8> { self.0.to_raw_vec() }
+		}
+
+		impl $crate::AppKey for Pair {
+			type UntypedGeneric = $pair;
+			type Public = Public;
+			type Pair = Pair;
+			type Signature = Signature;
+			const ID: $crate::KeyTypeId = $key_type;
+		}
+
+		impl $crate::AppPair for Pair {
+			type Generic = $pair;
+		}
+	};
+}
+
+/// Declares Public type which is functionally equivalent to `$public`, but is new
+/// Application-specific type whose identifier is `$key_type`.
+/// can only be used together with `full_crypto` feature
+/// For full functionality, app_crypto_public_common! must be called too.
+#[macro_export]
+macro_rules! app_crypto_public_full_crypto {
+	($public:ty, $key_type:expr) => {
+		$crate::wrap!{
+			/// A generic `AppPublic` wrapper type over $public crypto; this has no specific App.
+			#[derive(
+				Clone, Default, Eq, PartialEq, Ord, PartialOrd,
+				$crate::codec::Encode,
+				$crate::codec::Decode,
+				$crate::RuntimeDebug,
+			)]
+			#[derive(Hash)]
+			pub struct Public($public);
+		}
+
+		impl $crate::CryptoType for Public {
+			type Pair = Pair;
+		}
+
+		impl $crate::AppKey for Public {
+			type UntypedGeneric = $public;
+			type Public = Public;
+			type Pair = Pair;
+			type Signature = Signature;
+			const ID: $crate::KeyTypeId = $key_type;
+		}
+	}
+}
+
+/// Declares Public type which is functionally equivalent to `$public`, but is new
+/// Application-specific type whose identifier is `$key_type`.
+/// can only be used without `full_crypto` feature
+/// For full functionality, app_crypto_public_common! must be called too.
+#[macro_export]
+macro_rules! app_crypto_public_not_full_crypto {
+	($public:ty, $key_type:expr) => {
+		$crate::wrap!{
+			/// A generic `AppPublic` wrapper type over $public crypto; this has no specific App.
+			#[derive(
+				Clone, Default, Eq, PartialEq, Ord, PartialOrd,
+				$crate::codec::Encode,
+				$crate::codec::Decode,
+				$crate::RuntimeDebug,
+			)]
+			pub struct Public($public);
+		}
+
+		impl $crate::CryptoType for Public {}
+
+		impl $crate::AppKey for Public {
+			type UntypedGeneric = $public;
+			type Public = Public;
+			type Signature = Signature;
+			const ID: $crate::KeyTypeId = $key_type;
+		}
+	}
+}
+
+/// Declares Public type which is functionally equivalent to `$public`, but is new
+/// Application-specific type whose identifier is `$key_type`.
+/// For full functionality, app_crypto_public_(not)_full_crypto! must be called too.
+#[macro_export]
+macro_rules! app_crypto_public_common {
+	($public:ty, $sig:ty, $key_type:expr) => {
+		impl $crate::Derive for Public {
+			#[cfg(feature = "std")]
+			fn derive<Iter: Iterator<Item=$crate::DeriveJunction>>(&self,
+				path: Iter
+			) -> Option<Self> {
+				self.0.derive(path).map(Self)
+			}
+		}
+
+		#[cfg(feature = "std")]
+		impl std::fmt::Display for Public {
+			fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+				use $crate::Ss58Codec;
+				write!(f, "{}", self.0.to_ss58check())
+			}
+		}
+		#[cfg(feature = "std")]
+		impl $crate::serde::Serialize for Public {
+			fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error> where
+				S: $crate::serde::Serializer
+			{
+				use $crate::Ss58Codec;
+				serializer.serialize_str(&self.to_ss58check())
+			}
+		}
+		#[cfg(feature = "std")]
+		impl<'de> $crate::serde::Deserialize<'de> for Public {
+			fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error> where
+				D: $crate::serde::Deserializer<'de>
+			{
+				use $crate::Ss58Codec;
+				Public::from_ss58check(&String::deserialize(deserializer)?)
+					.map_err(|e| $crate::serde::de::Error::custom(format!("{:?}", e)))
+			}
+		}
+
+		impl AsRef<[u8]> for Public {
+			fn as_ref(&self) -> &[u8] { self.0.as_ref() }
+		}
+
+		impl AsMut<[u8]> for Public {
+			fn as_mut(&mut self) -> &mut [u8] { self.0.as_mut() }
+		}
+
+		impl $crate::Public for Public {
+			fn from_slice(x: &[u8]) -> Self { Self(<$public>::from_slice(x)) }
+		}
+
+		impl $crate::AppPublic for Public {
+			type Generic = $public;
+		}
+
+		impl $crate::RuntimeAppPublic for Public where $public: $crate::RuntimePublic<Signature=$sig> {
+			const ID: $crate::KeyTypeId = $key_type;
+			type Signature = Signature;
+
+			fn all() -> $crate::Vec<Self> {
+				<$public as $crate::RuntimePublic>::all($key_type).into_iter().map(Self).collect()
+			}
+
+			fn generate_pair(seed: Option<$crate::Vec<u8>>) -> Self {
+				Self(<$public as $crate::RuntimePublic>::generate_pair($key_type, seed))
+			}
+
+			fn sign<M: AsRef<[u8]>>(&self, msg: &M) -> Option<Self::Signature> {
+				<$public as $crate::RuntimePublic>::sign(
+					self.as_ref(),
+					$key_type,
+					msg,
+				).map(Signature)
+			}
+
+			fn verify<M: AsRef<[u8]>>(&self, msg: &M, signature: &Self::Signature) -> bool {
+				<$public as $crate::RuntimePublic>::verify(self.as_ref(), msg, &signature.as_ref())
+			}
+		}
+	}
+}
+
+/// Declares Signature type which is functionally equivalent to `$sig`, but is new
+/// Application-specific type whose identifier is `$key_type`.
+/// can only be used together with `full_crypto` feature
+/// For full functionality, app_crypto_public_common! must be called too.
+#[macro_export]
+macro_rules! app_crypto_signature_full_crypto {
+	($sig:ty, $key_type:expr) => {
+		$crate::wrap! {
+			/// A generic `AppPublic` wrapper type over $public crypto; this has no specific App.
+			#[derive(Clone, Default, Eq, PartialEq,
+				$crate::codec::Encode,
+				$crate::codec::Decode,
+				$crate::RuntimeDebug,
+			)]
+			#[derive(Hash)]
+			pub struct Signature($sig);
+		}
+
+		impl $crate::CryptoType for Signature {
+			type Pair = Pair;
+		}
+
+		impl $crate::AppKey for Signature {
+			type UntypedGeneric = $sig;
+			type Public = Public;
+			type Pair = Pair;
+			type Signature = Signature;
+			const ID: $crate::KeyTypeId = $key_type;
+		}
+	}
+}
+
+/// Declares Signature type which is functionally equivalent to `$sig`, but is new
+/// Application-specific type whose identifier is `$key_type`.
+/// can only be used without `full_crypto` feature
+/// For full functionality, app_crypto_public_common! must be called too.
+#[macro_export]
+macro_rules! app_crypto_signature_not_full_crypto {
+	($sig:ty, $key_type:expr) => {
+		$crate::wrap! {
+			/// A generic `AppPublic` wrapper type over $public crypto; this has no specific App.
+			#[derive(Clone, Default, Eq, PartialEq,
+				$crate::codec::Encode,
+				$crate::codec::Decode,
+				$crate::RuntimeDebug,
+			)]
+			pub struct Signature($sig);
+		}
+		
+		impl $crate::CryptoType for Signature {}
+
+		impl $crate::AppKey for Signature {
+			type UntypedGeneric = $sig;
+			type Public = Public;
+			type Signature = Signature;
+			const ID: $crate::KeyTypeId = $key_type;
+		}
+	}
+}
+
+/// Declares Signature type which is functionally equivalent to `$sig`, but is new
+/// Application-specific type whose identifier is `$key_type`.
+/// For full functionality, app_crypto_public_(not)_full_crypto! must be called too.
+#[macro_export]
+macro_rules! app_crypto_signature_common {
+	($sig:ty, $key_type:expr) => {
+		impl $crate::Deref for Signature {
+			type Target = [u8];
+
+			fn deref(&self) -> &Self::Target { self.0.as_ref() }
+		}
+
+		impl AsRef<[u8]> for Signature {
+			fn as_ref(&self) -> &[u8] { self.0.as_ref() }
+		}
+
+		impl $crate::AppSignature for Signature {
+			type Generic = $sig;
+		}
+	}
+}
+
+/// Implement bidirectional `From` and on-way `AsRef`/`AsMut` for two types, `$inner` and `$outer`.
+///
+/// ```rust
+/// substrate_application_crypto::wrap! {
+///     pub struct Wrapper(u32);
+/// }
+/// ```
+#[macro_export]
+macro_rules! wrap {
+	($( #[ $attr:meta ] )* struct $outer:ident($inner:ty);) => {
+		$( #[ $attr ] )*
+		struct $outer( $inner );
+		$crate::wrap!($inner, $outer);
+	};
+	($( #[ $attr:meta ] )* pub struct $outer:ident($inner:ty);) => {
+		$( #[ $attr ] )*
+		pub struct $outer( $inner );
+		$crate::wrap!($inner, $outer);
+	};
+	($inner:ty, $outer:ty) => {
+		impl $crate::Wraps for $outer {
+			type Inner = $inner;
+		}
+		impl From<$inner> for $outer {
+			fn from(inner: $inner) -> Self {
+				Self(inner)
+			}
+		}
+		impl From<$outer> for $inner {
+			fn from(outer: $outer) -> Self {
+				outer.0
+			}
+		}
+		impl AsRef<$inner> for $outer {
+			fn as_ref(&self) -> &$inner {
+				&self.0
+			}
+		}
+		impl AsMut<$inner> for $outer {
+			fn as_mut(&mut self) -> &mut $inner {
+				&mut self.0
+			}
+		}
+	}
+}

substrate/primitives/application-crypto/src/sr25519.rs

@@ -0,0 +1,80 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Sr25519 crypto types.
+
+use crate::{RuntimePublic, KeyTypeId};
+
+use rstd::vec::Vec;
+
+pub use primitives::sr25519::*;
+
+mod app {
+	use primitives::testing::SR25519;
+	crate::app_crypto!(super, SR25519);
+
+	impl crate::traits::BoundToRuntimeAppPublic for Public {
+		type Public = Self;
+	}
+}
+
+pub use app::{Public as AppPublic, Signature as AppSignature};
+#[cfg(feature = "full_crypto")]
+pub use app::Pair as AppPair;
+
+impl RuntimePublic for Public {
+	type Signature = Signature;
+
+	fn all(key_type: KeyTypeId) -> crate::Vec<Self> {
+		runtime_io::crypto::sr25519_public_keys(key_type)
+	}
+
+	fn generate_pair(key_type: KeyTypeId, seed: Option<Vec<u8>>) -> Self {
+		runtime_io::crypto::sr25519_generate(key_type, seed)
+	}
+
+	fn sign<M: AsRef<[u8]>>(&self, key_type: KeyTypeId, msg: &M) -> Option<Self::Signature> {
+		runtime_io::crypto::sr25519_sign(key_type, self, msg.as_ref())
+	}
+
+	fn verify<M: AsRef<[u8]>>(&self, msg: &M, signature: &Self::Signature) -> bool {
+		runtime_io::crypto::sr25519_verify(&signature, msg.as_ref(), self)
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use sr_primitives::{generic::BlockId, traits::ProvideRuntimeApi};
+	use primitives::{testing::{KeyStore, SR25519}, crypto::Pair};
+	use test_client::{
+		TestClientBuilder, DefaultTestClientBuilderExt, TestClientBuilderExt,
+		runtime::{TestAPI, app_crypto::sr25519::{AppPair, AppPublic}},
+	};
+
+	#[test]
+	fn sr25519_works_in_runtime() {
+		let keystore = KeyStore::new();
+		let test_client = TestClientBuilder::new().set_keystore(keystore.clone()).build();
+		let (signature, public) = test_client.runtime_api()
+			.test_sr25519_crypto(&BlockId::Number(0))
+			.expect("Tests `sr25519` crypto.");
+
+		let key_pair = keystore.read().sr25519_key_pair(SR25519, public.as_ref())
+			.expect("There should be at a `sr25519` key in the keystore for the given public key.");
+
+		assert!(AppPair::verify(&signature, "sr25519", &AppPublic::from(key_pair.public())));
+	}
+}

substrate/primitives/application-crypto/src/traits.rs

@@ -0,0 +1,145 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+#[cfg(feature = "full_crypto")]
+use primitives::crypto::Pair;
+
+use codec::Codec;
+use primitives::crypto::{KeyTypeId, CryptoType, IsWrappedBy, Public};
+use rstd::{fmt::Debug, vec::Vec};
+
+/// An application-specific key.
+pub trait AppKey: 'static + Send + Sync + Sized + CryptoType + Clone {
+	/// The corresponding type as a generic crypto type.
+	type UntypedGeneric: IsWrappedBy<Self>;
+
+	/// The corresponding public key type in this application scheme.
+	type Public: AppPublic;
+
+	/// The corresponding key pair type in this application scheme.
+	#[cfg(feature = "full_crypto")]
+	type Pair: AppPair;
+
+	/// The corresponding signature type in this application scheme.
+	type Signature: AppSignature;
+
+	/// An identifier for this application-specific key type.
+	const ID: KeyTypeId;
+}
+
+/// Type which implements Hash in std, not when no-std (std variant).
+#[cfg(feature = "std")]
+pub trait MaybeHash: rstd::hash::Hash {}
+#[cfg(feature = "std")]
+impl<T: rstd::hash::Hash> MaybeHash for T {}
+
+/// Type which implements Hash in std, not when no-std (no-std variant).
+#[cfg(all(not(feature = "std"), not(feature = "full_crypto")))]
+pub trait MaybeHash {}
+#[cfg(all(not(feature = "std"), not(feature = "full_crypto")))]
+impl<T> MaybeHash for T {}
+
+/// Type which implements Debug and Hash in std, not when no-std (no-std variant with crypto).
+#[cfg(all(not(feature = "std"), feature = "full_crypto"))]
+pub trait MaybeDebugHash: rstd::hash::Hash  {}
+#[cfg(all(not(feature = "std"), feature = "full_crypto"))]
+impl<T: rstd::hash::Hash> MaybeDebugHash for T {}
+
+/// A application's public key.
+pub trait AppPublic:
+	AppKey + Public + Ord + PartialOrd + Eq + PartialEq + Debug + MaybeHash + codec::Codec
+{
+	/// The wrapped type which is just a plain instance of `Public`.
+	type Generic:
+		IsWrappedBy<Self> + Public + Ord + PartialOrd + Eq + PartialEq + Debug + MaybeHash + codec::Codec;
+}
+
+/// A application's key pair.
+#[cfg(feature = "full_crypto")]
+pub trait AppPair: AppKey + Pair<Public=<Self as AppKey>::Public> {
+	/// The wrapped type which is just a plain instance of `Pair`.
+	type Generic: IsWrappedBy<Self> + Pair<Public=<<Self as AppKey>::Public as AppPublic>::Generic>;
+}
+
+/// A application's signature.
+pub trait AppSignature: AppKey + Eq + PartialEq + Debug + MaybeHash {
+	/// The wrapped type which is just a plain instance of `Signature`.
+	type Generic: IsWrappedBy<Self> + Eq + PartialEq + Debug + MaybeHash;
+}
+
+/// A runtime interface for a public key.
+pub trait RuntimePublic: Sized {
+	/// The signature that will be generated when signing with the corresponding private key.
+	type Signature: Codec + Debug + MaybeHash + Eq + PartialEq + Clone;
+
+	/// Returns all public keys for the given key type in the keystore.
+	fn all(key_type: KeyTypeId) -> crate::Vec<Self>;
+
+	/// Generate a public/private pair for the given key type with an optional `seed` and
+	/// store it in the keystore.
+	///
+	/// The `seed` needs to be valid utf8.
+	///
+	/// Returns the generated public key.
+	fn generate_pair(key_type: KeyTypeId, seed: Option<Vec<u8>>) -> Self;
+
+	/// Sign the given message with the corresponding private key of this public key.
+	///
+	/// The private key will be requested from the keystore using the given key type.
+	///
+	/// Returns the signature or `None` if the private key could not be found or some other error
+	/// occurred.
+	fn sign<M: AsRef<[u8]>>(&self, key_type: KeyTypeId, msg: &M) -> Option<Self::Signature>;
+
+	/// Verify that the given signature matches the given message using this public key.
+	fn verify<M: AsRef<[u8]>>(&self, msg: &M, signature: &Self::Signature) -> bool;
+}
+
+/// A runtime interface for an application's public key.
+pub trait RuntimeAppPublic: Sized  {
+	/// An identifier for this application-specific key type.
+	const ID: KeyTypeId;
+
+	/// The signature that will be generated when signing with the corresponding private key.
+	type Signature: Codec + Debug + MaybeHash + Eq + PartialEq + Clone;
+
+	/// Returns all public keys for this application in the keystore.
+	fn all() -> crate::Vec<Self>;
+
+	/// Generate a public/private pair with an optional `seed` and store it in the keystore.
+	///
+	/// The `seed` needs to be valid utf8.
+	///
+	/// Returns the generated public key.
+	fn generate_pair(seed: Option<Vec<u8>>) -> Self;
+
+	/// Sign the given message with the corresponding private key of this public key.
+	///
+	/// The private key will be requested from the keystore.
+	///
+	/// Returns the signature or `None` if the private key could not be found or some other error
+	/// occurred.
+	fn sign<M: AsRef<[u8]>>(&self, msg: &M) -> Option<Self::Signature>;
+
+	/// Verify that the given signature matches the given message using this public key.
+	fn verify<M: AsRef<[u8]>>(&self, msg: &M, signature: &Self::Signature) -> bool;
+}
+
+/// Something that bound to a fixed `RuntimeAppPublic`.
+pub trait BoundToRuntimeAppPublic {
+	/// The `RuntimeAppPublic` this type is bound to.
+	type Public: RuntimeAppPublic;
+}

substrate/primitives/authority-discovery/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "substrate-authority-discovery-primitives"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Authority discovery primitives"
+edition = "2018"
+
+[dependencies]
+app-crypto = { package = "substrate-application-crypto", path = "../application-crypto", default-features = false }
+codec = { package = "parity-scale-codec", default-features = false, version = "1.0.3" }
+rstd = { package = "sr-std", path = "../sr-std", default-features = false }
+sr-api = { path = "../sr-api", default-features = false }
+sr-primitives = { path = "../sr-primitives", default-features = false }
+
+[features]
+default = ["std"]
+std = [
+	"app-crypto/std",
+	"codec/std",
+	"rstd/std",
+	"sr-api/std",
+	"sr-primitives/std"
+]

substrate/primitives/authority-discovery/src/lib.rs

@@ -0,0 +1,47 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Runtime Api to help discover authorities.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+use rstd::vec::Vec;
+
+mod app {
+	use app_crypto::{app_crypto, key_types::AUTHORITY_DISCOVERY, sr25519};
+	app_crypto!(sr25519, AUTHORITY_DISCOVERY);
+}
+
+/// An authority discovery authority keypair.
+#[cfg(feature = "std")]
+pub type AuthorityPair = app::Pair;
+
+/// An authority discovery authority identifier.
+pub type AuthorityId = app::Public;
+
+/// An authority discovery authority signature.
+pub type AuthoritySignature = app::Signature;
+
+sr_api::decl_runtime_apis! {
+	/// The authority discovery api.
+	///
+	/// This api is used by the `core/authority-discovery` module to retrieve identifiers
+	/// of the current authority set.
+	pub trait AuthorityDiscoveryApi {
+		/// Retrieve authority identifiers of the current authority set.
+		fn authorities() -> Vec<AuthorityId>;
+	}
+}

substrate/primitives/block-builder/runtime-api/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "substrate-block-builder-runtime-api"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+sr-primitives = { path = "../../sr-primitives", default-features = false }
+sr-api = { path = "../../sr-api", default-features = false }
+rstd = { package = "sr-std", path = "../../sr-std", default-features = false }
+inherents = { package = "substrate-inherents", path = "../../inherents", default-features = false }
+
+[features]
+default = [ "std" ]
+std = [
+	"sr-primitives/std",
+	"inherents/std",
+	"sr-api/std",
+	"rstd/std",
+]

substrate/primitives/block-builder/runtime-api/src/lib.rs

@@ -0,0 +1,43 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The block builder runtime api.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+use sr_primitives::{traits::Block as BlockT, ApplyResult};
+
+use inherents::{InherentData, CheckInherentsResult};
+
+sr_api::decl_runtime_apis! {
+	/// The `BlockBuilder` api trait that provides the required functionality for building a block.
+	#[api_version(3)]
+	pub trait BlockBuilder {
+		/// Apply the given extrinsics.
+		fn apply_extrinsic(extrinsic: <Block as BlockT>::Extrinsic) -> ApplyResult;
+		/// Finish the current block.
+		#[renamed("finalise_block", 3)]
+		fn finalize_block() -> <Block as BlockT>::Header;
+		/// Generate inherent extrinsics. The inherent data will vary from chain to chain.
+		fn inherent_extrinsics(
+			inherent: InherentData,
+		) -> rstd::vec::Vec<<Block as BlockT>::Extrinsic>;
+		/// Check that the inherents are valid. The inherent data will vary from chain to chain.
+		fn check_inherents(block: Block, data: InherentData) -> CheckInherentsResult;
+		/// Generate a random seed.
+		fn random_seed() -> <Block as BlockT>::Hash;
+	}
+}

substrate/primitives/consensus/aura/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "substrate-consensus-aura-primitives"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Primitives for Aura consensus"
+edition = "2018"
+
+[dependencies]
+app-crypto = { package = "substrate-application-crypto", path = "../../application-crypto", default-features = false }
+codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false }
+rstd = { package = "sr-std", path = "../../sr-std", default-features = false }
+sr-api = { path = "../../sr-api", default-features = false }
+sr-primitives = { path = "../../sr-primitives", default-features = false }
+
+[features]
+default = ["std"]
+std = [
+	"app-crypto/std",
+	"codec/std",
+	"rstd/std",
+	"sr-api/std",
+	"sr-primitives/std",
+]

substrate/primitives/consensus/aura/src/lib.rs

@@ -0,0 +1,89 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Primitives for Aura.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+use codec::{Encode, Decode, Codec};
+use rstd::vec::Vec;
+use sr_primitives::ConsensusEngineId;
+
+pub mod sr25519 {
+	mod app_sr25519 {
+		use app_crypto::{app_crypto, key_types::AURA, sr25519};
+		app_crypto!(sr25519, AURA);
+	}
+
+	/// An Aura authority keypair using S/R 25519 as its crypto.
+	#[cfg(feature = "std")]
+	pub type AuthorityPair = app_sr25519::Pair;
+
+	/// An Aura authority signature using S/R 25519 as its crypto.
+	pub type AuthoritySignature = app_sr25519::Signature;
+
+	/// An Aura authority identifier using S/R 25519 as its crypto.
+	pub type AuthorityId = app_sr25519::Public;
+}
+
+pub mod ed25519 {
+	mod app_ed25519 {
+		use app_crypto::{app_crypto, key_types::AURA, ed25519};
+		app_crypto!(ed25519, AURA);
+	}
+
+	/// An Aura authority keypair using Ed25519 as its crypto.
+	#[cfg(feature = "std")]
+	pub type AuthorityPair = app_ed25519::Pair;
+
+	/// An Aura authority signature using Ed25519 as its crypto.
+	pub type AuthoritySignature = app_ed25519::Signature;
+
+	/// An Aura authority identifier using Ed25519 as its crypto.
+	pub type AuthorityId = app_ed25519::Public;
+}
+
+/// The `ConsensusEngineId` of AuRa.
+pub const AURA_ENGINE_ID: ConsensusEngineId = [b'a', b'u', b'r', b'a'];
+
+/// The index of an authority.
+pub type AuthorityIndex = u32;
+
+/// An consensus log item for Aura.
+#[derive(Decode, Encode)]
+pub enum ConsensusLog<AuthorityId: Codec> {
+	/// The authorities have changed.
+	#[codec(index = "1")]
+	AuthoritiesChange(Vec<AuthorityId>),
+	/// Disable the authority with given index.
+	#[codec(index = "2")]
+	OnDisabled(AuthorityIndex),
+}
+
+sr_api::decl_runtime_apis! {
+	/// API necessary for block authorship with aura.
+	pub trait AuraApi<AuthorityId: Codec> {
+		/// Return the slot duration in seconds for Aura.
+		/// Currently, only the value provided by this type at genesis
+		/// will be used.
+		///
+		/// Dynamic slot duration may be supported in the future.
+		fn slot_duration() -> u64;
+
+		// Return the current set of authorities.
+		fn authorities() -> Vec<AuthorityId>;
+	}
+}

substrate/primitives/consensus/babe/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "substrate-consensus-babe-primitives"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Primitives for BABE consensus"
+edition = "2018"
+
+[dependencies]
+app-crypto = { package = "substrate-application-crypto", path = "../../application-crypto", default-features = false }
+codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false }
+rstd = { package = "sr-std", path = "../../sr-std", default-features = false }
+schnorrkel = { version = "0.8.5", features = ["preaudit_deprecated"], optional = true }
+slots = { package = "substrate-consensus-slots", path = "../../../client/consensus/slots", optional = true }
+sr-api = { path = "../../sr-api", default-features = false }
+sr-primitives = {  path = "../../sr-primitives", default-features = false }
+
+[features]
+default = ["std"]
+std = [
+	"app-crypto/std",
+	"codec/std",
+	"rstd/std",
+	"schnorrkel",
+	"slots",
+	"sr-api/std",
+	"sr-primitives/std",
+]

substrate/primitives/consensus/babe/src/digest.rs

@@ -0,0 +1,284 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Private implementation details of BABE digests.
+
+#[cfg(feature = "std")]
+use super::{BABE_ENGINE_ID, AuthoritySignature};
+#[cfg(not(feature = "std"))]
+use super::{VRF_OUTPUT_LENGTH, VRF_PROOF_LENGTH};
+use super::{AuthorityId, AuthorityIndex, SlotNumber, BabeAuthorityWeight};
+#[cfg(feature = "std")]
+use sr_primitives::{DigestItem, generic::OpaqueDigestItemId};
+#[cfg(feature = "std")]
+use std::fmt::Debug;
+use codec::{Decode, Encode};
+#[cfg(feature = "std")]
+use codec::{Codec, Input, Error};
+#[cfg(feature = "std")]
+use schnorrkel::{
+	SignatureError, errors::MultiSignatureStage,
+	vrf::{VRFProof, VRFOutput, VRF_OUTPUT_LENGTH, VRF_PROOF_LENGTH}
+};
+use rstd::vec::Vec;
+
+
+/// A BABE pre-runtime digest. This contains all data required to validate a
+/// block and for the BABE runtime module. Slots can be assigned to a primary
+/// (VRF based) and to a secondary (slot number based).
+#[cfg(feature = "std")]
+#[derive(Clone, Debug)]
+pub enum BabePreDigest {
+	/// A primary VRF-based slot assignment.
+	Primary {
+		/// VRF output
+		vrf_output: VRFOutput,
+		/// VRF proof
+		vrf_proof: VRFProof,
+		/// Authority index
+		authority_index: super::AuthorityIndex,
+		/// Slot number
+		slot_number: SlotNumber,
+	},
+	/// A secondary deterministic slot assignment.
+	Secondary {
+		/// Authority index
+		authority_index: super::AuthorityIndex,
+		/// Slot number
+		slot_number: SlotNumber,
+	},
+}
+
+#[cfg(feature = "std")]
+impl BabePreDigest {
+	/// Returns the slot number of the pre digest.
+	pub fn authority_index(&self) -> AuthorityIndex {
+		match self {
+			BabePreDigest::Primary { authority_index, .. } => *authority_index,
+			BabePreDigest::Secondary { authority_index, .. } => *authority_index,
+		}
+	}
+
+	/// Returns the slot number of the pre digest.
+	pub fn slot_number(&self) -> SlotNumber {
+		match self {
+			BabePreDigest::Primary { slot_number, .. } => *slot_number,
+			BabePreDigest::Secondary { slot_number, .. } => *slot_number,
+		}
+	}
+
+	/// Returns the weight _added_ by this digest, not the cumulative weight
+	/// of the chain.
+	pub fn added_weight(&self) -> crate::BabeBlockWeight {
+		match self {
+			BabePreDigest::Primary { .. } => 1,
+			BabePreDigest::Secondary { .. } => 0,
+		}
+	}
+}
+
+/// The prefix used by BABE for its VRF keys.
+pub const BABE_VRF_PREFIX: &[u8] = b"substrate-babe-vrf";
+
+/// A raw version of `BabePreDigest`, usable on `no_std`.
+#[derive(Copy, Clone, Encode, Decode)]
+pub enum RawBabePreDigest {
+	/// A primary VRF-based slot assignment.
+	#[codec(index = "1")]
+	Primary {
+		/// Authority index
+		authority_index: AuthorityIndex,
+		/// Slot number
+		slot_number: SlotNumber,
+		/// VRF output
+		vrf_output: [u8; VRF_OUTPUT_LENGTH],
+		/// VRF proof
+		vrf_proof: [u8; VRF_PROOF_LENGTH],
+	},
+	/// A secondary deterministic slot assignment.
+	#[codec(index = "2")]
+	Secondary {
+		/// Authority index
+		///
+		/// This is not strictly-speaking necessary, since the secondary slots
+		/// are assigned based on slot number and epoch randomness. But including
+		/// it makes things easier for higher-level users of the chain data to
+		/// be aware of the author of a secondary-slot block.
+		authority_index: AuthorityIndex,
+		/// Slot number
+		slot_number: SlotNumber,
+	},
+}
+
+impl RawBabePreDigest {
+	/// Returns the slot number of the pre digest.
+	pub fn slot_number(&self) -> SlotNumber {
+		match self {
+			RawBabePreDigest::Primary { slot_number, .. } => *slot_number,
+			RawBabePreDigest::Secondary { slot_number, .. } => *slot_number,
+		}
+	}
+}
+
+#[cfg(feature = "std")]
+impl Encode for BabePreDigest {
+	fn encode(&self) -> Vec<u8> {
+		let raw = match self {
+			BabePreDigest::Primary {
+				vrf_output,
+				vrf_proof,
+				authority_index,
+				slot_number,
+			} => {
+				RawBabePreDigest::Primary {
+					vrf_output: *vrf_output.as_bytes(),
+					vrf_proof: vrf_proof.to_bytes(),
+					authority_index: *authority_index,
+					slot_number: *slot_number,
+				}
+			},
+			BabePreDigest::Secondary {
+				authority_index,
+				slot_number,
+			} => {
+				RawBabePreDigest::Secondary {
+					authority_index: *authority_index,
+					slot_number: *slot_number,
+				}
+			},
+		};
+
+		codec::Encode::encode(&raw)
+	}
+}
+
+#[cfg(feature = "std")]
+impl codec::EncodeLike for BabePreDigest {}
+
+#[cfg(feature = "std")]
+impl Decode for BabePreDigest {
+	fn decode<R: Input>(i: &mut R) -> Result<Self, Error> {
+		let pre_digest = match Decode::decode(i)? {
+			RawBabePreDigest::Primary { vrf_output, vrf_proof, authority_index, slot_number } => {
+				// Verify (at compile time) that the sizes in babe_primitives are correct
+				let _: [u8; super::VRF_OUTPUT_LENGTH] = vrf_output;
+				let _: [u8; super::VRF_PROOF_LENGTH] = vrf_proof;
+
+				BabePreDigest::Primary {
+					vrf_proof: VRFProof::from_bytes(&vrf_proof).map_err(convert_error)?,
+					vrf_output: VRFOutput::from_bytes(&vrf_output).map_err(convert_error)?,
+					authority_index,
+					slot_number,
+				}
+			},
+			RawBabePreDigest::Secondary { authority_index, slot_number } => {
+				BabePreDigest::Secondary { authority_index, slot_number }
+			},
+		};
+
+		Ok(pre_digest)
+	}
+}
+
+/// Information about the next epoch. This is broadcast in the first block
+/// of the epoch.
+#[derive(Decode, Encode, Default, PartialEq, Eq, Clone, sr_primitives::RuntimeDebug)]
+pub struct NextEpochDescriptor {
+	/// The authorities.
+	pub authorities: Vec<(AuthorityId, BabeAuthorityWeight)>,
+
+	/// The value of randomness to use for the slot-assignment.
+	pub randomness: [u8; VRF_OUTPUT_LENGTH],
+}
+
+/// A digest item which is usable with BABE consensus.
+#[cfg(feature = "std")]
+pub trait CompatibleDigestItem: Sized {
+	/// Construct a digest item which contains a BABE pre-digest.
+	fn babe_pre_digest(seal: BabePreDigest) -> Self;
+
+	/// If this item is an BABE pre-digest, return it.
+	fn as_babe_pre_digest(&self) -> Option<BabePreDigest>;
+
+	/// Construct a digest item which contains a BABE seal.
+	fn babe_seal(signature: AuthoritySignature) -> Self;
+
+	/// If this item is a BABE signature, return the signature.
+	fn as_babe_seal(&self) -> Option<AuthoritySignature>;
+
+	/// If this item is a BABE epoch, return it.
+	fn as_next_epoch_descriptor(&self) -> Option<NextEpochDescriptor>;
+}
+
+#[cfg(feature = "std")]
+impl<Hash> CompatibleDigestItem for DigestItem<Hash> where
+	Hash: Debug + Send + Sync + Eq + Clone + Codec + 'static
+{
+	fn babe_pre_digest(digest: BabePreDigest) -> Self {
+		DigestItem::PreRuntime(BABE_ENGINE_ID, digest.encode())
+	}
+
+	fn as_babe_pre_digest(&self) -> Option<BabePreDigest> {
+		self.try_to(OpaqueDigestItemId::PreRuntime(&BABE_ENGINE_ID))
+	}
+
+	fn babe_seal(signature: AuthoritySignature) -> Self {
+		DigestItem::Seal(BABE_ENGINE_ID, signature.encode())
+	}
+
+	fn as_babe_seal(&self) -> Option<AuthoritySignature> {
+		self.try_to(OpaqueDigestItemId::Seal(&BABE_ENGINE_ID))
+	}
+
+	fn as_next_epoch_descriptor(&self) -> Option<NextEpochDescriptor> {
+		self.try_to(OpaqueDigestItemId::Consensus(&BABE_ENGINE_ID))
+			.and_then(|x: super::ConsensusLog| match x {
+				super::ConsensusLog::NextEpochData(n) => Some(n),
+				_ => None,
+			})
+	}
+}
+
+#[cfg(feature = "std")]
+fn convert_error(e: SignatureError) -> codec::Error {
+	use SignatureError::*;
+	use MultiSignatureStage::*;
+	match e {
+		EquationFalse => "Signature error: `EquationFalse`".into(),
+		PointDecompressionError => "Signature error: `PointDecompressionError`".into(),
+		ScalarFormatError => "Signature error: `ScalarFormatError`".into(),
+		NotMarkedSchnorrkel => "Signature error: `NotMarkedSchnorrkel`".into(),
+		BytesLengthError { .. } => "Signature error: `BytesLengthError`".into(),
+		MuSigAbsent { musig_stage: Commitment } =>
+			"Signature error: `MuSigAbsent` at stage `Commitment`".into(),
+		MuSigAbsent { musig_stage: Reveal } =>
+			"Signature error: `MuSigAbsent` at stage `Reveal`".into(),
+		MuSigAbsent { musig_stage: Cosignature } =>
+			"Signature error: `MuSigAbsent` at stage `Commitment`".into(),
+		MuSigInconsistent { musig_stage: Commitment, duplicate: true } =>
+			"Signature error: `MuSigInconsistent` at stage `Commitment` on duplicate".into(),
+		MuSigInconsistent { musig_stage: Commitment, duplicate: false } =>
+			"Signature error: `MuSigInconsistent` at stage `Commitment` on not duplicate".into(),
+		MuSigInconsistent { musig_stage: Reveal, duplicate: true } =>
+			"Signature error: `MuSigInconsistent` at stage `Reveal` on duplicate".into(),
+		MuSigInconsistent { musig_stage: Reveal, duplicate: false } =>
+			"Signature error: `MuSigInconsistent` at stage `Reveal` on not duplicate".into(),
+		MuSigInconsistent { musig_stage: Cosignature, duplicate: true } =>
+			"Signature error: `MuSigInconsistent` at stage `Cosignature` on duplicate".into(),
+		MuSigInconsistent { musig_stage: Cosignature, duplicate: false } =>
+			"Signature error: `MuSigInconsistent` at stage `Cosignature` on not duplicate".into(),
+	}
+}

substrate/primitives/consensus/babe/src/lib.rs

@@ -0,0 +1,176 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Primitives for BABE.
+#![deny(warnings)]
+#![forbid(unsafe_code, missing_docs, unused_variables, unused_imports)]
+#![cfg_attr(not(feature = "std"), no_std)]
+
+mod digest;
+
+use codec::{Encode, Decode};
+use rstd::vec::Vec;
+use sr_primitives::{ConsensusEngineId, RuntimeDebug};
+
+#[cfg(feature = "std")]
+pub use digest::{BabePreDigest, CompatibleDigestItem};
+pub use digest::{BABE_VRF_PREFIX, RawBabePreDigest, NextEpochDescriptor};
+
+mod app {
+	use app_crypto::{app_crypto, key_types::BABE, sr25519};
+	app_crypto!(sr25519, BABE);
+}
+
+/// A Babe authority keypair. Necessarily equivalent to the schnorrkel public key used in
+/// the main Babe module. If that ever changes, then this must, too.
+#[cfg(feature = "std")]
+pub type AuthorityPair = app::Pair;
+
+/// A Babe authority signature.
+pub type AuthoritySignature = app::Signature;
+
+/// A Babe authority identifier. Necessarily equivalent to the schnorrkel public key used in
+/// the main Babe module. If that ever changes, then this must, too.
+pub type AuthorityId = app::Public;
+
+/// The `ConsensusEngineId` of BABE.
+pub const BABE_ENGINE_ID: ConsensusEngineId = *b"BABE";
+
+/// The length of the VRF output
+pub const VRF_OUTPUT_LENGTH: usize = 32;
+
+/// The length of the VRF proof
+pub const VRF_PROOF_LENGTH: usize = 64;
+
+/// The length of the public key
+pub const PUBLIC_KEY_LENGTH: usize = 32;
+
+/// How many blocks to wait before running the median algorithm for relative time
+/// This will not vary from chain to chain as it is not dependent on slot duration
+/// or epoch length.
+pub const MEDIAN_ALGORITHM_CARDINALITY: usize = 1200; // arbitrary suggestion by w3f-research.
+
+/// The index of an authority.
+pub type AuthorityIndex = u32;
+
+/// A slot number.
+pub type SlotNumber = u64;
+
+/// The weight of an authority.
+// NOTE: we use a unique name for the weight to avoid conflicts with other
+//       `Weight` types, since the metadata isn't able to disambiguate.
+pub type BabeAuthorityWeight = u64;
+
+/// The weight of a BABE block.
+pub type BabeBlockWeight = u32;
+
+/// BABE epoch information
+#[derive(Decode, Encode, Default, PartialEq, Eq, Clone, RuntimeDebug)]
+pub struct Epoch {
+	/// The epoch index
+	pub epoch_index: u64,
+	/// The starting slot of the epoch,
+	pub start_slot: SlotNumber,
+	/// The duration of this epoch
+	pub duration: SlotNumber,
+	/// The authorities and their weights
+	pub authorities: Vec<(AuthorityId, BabeAuthorityWeight)>,
+	/// Randomness for this epoch
+	pub randomness: [u8; VRF_OUTPUT_LENGTH],
+}
+
+impl Epoch {
+	/// "increment" the epoch, with given descriptor for the next.
+	pub fn increment(&self, descriptor: NextEpochDescriptor) -> Epoch {
+		Epoch {
+			epoch_index: self.epoch_index + 1,
+			start_slot: self.start_slot + self.duration,
+			duration: self.duration,
+			authorities: descriptor.authorities,
+			randomness: descriptor.randomness,
+		}
+	}
+
+	/// Produce the "end slot" of the epoch. This is NOT inclusive to the epoch,
+	// i.e. the slots covered by the epoch are `self.start_slot .. self.end_slot()`.
+	pub fn end_slot(&self) -> SlotNumber {
+		self.start_slot + self.duration
+	}
+}
+
+/// An consensus log item for BABE.
+#[derive(Decode, Encode, Clone, PartialEq, Eq)]
+pub enum ConsensusLog {
+	/// The epoch has changed. This provides information about the _next_
+	/// epoch - information about the _current_ epoch (i.e. the one we've just
+	/// entered) should already be available earlier in the chain.
+	#[codec(index = "1")]
+	NextEpochData(NextEpochDescriptor),
+	/// Disable the authority with given index.
+	#[codec(index = "2")]
+	OnDisabled(AuthorityIndex),
+}
+
+/// Configuration data used by the BABE consensus engine.
+#[derive(Clone, PartialEq, Eq, Encode, Decode, RuntimeDebug)]
+pub struct BabeConfiguration {
+	/// The slot duration in milliseconds for BABE. Currently, only
+	/// the value provided by this type at genesis will be used.
+	///
+	/// Dynamic slot duration may be supported in the future.
+	pub slot_duration: u64,
+
+	/// The duration of epochs in slots.
+	pub epoch_length: SlotNumber,
+
+	/// A constant value that is used in the threshold calculation formula.
+	/// Expressed as a rational where the first member of the tuple is the
+	/// numerator and the second is the denominator. The rational should
+	/// represent a value between 0 and 1.
+	/// In the threshold formula calculation, `1 - c` represents the probability
+	/// of a slot being empty.
+	pub c: (u64, u64),
+
+	/// The authorities for the genesis epoch.
+	pub genesis_authorities: Vec<(AuthorityId, BabeAuthorityWeight)>,
+
+	/// The randomness for the genesis epoch.
+	pub randomness: [u8; VRF_OUTPUT_LENGTH],
+
+	/// Whether this chain should run with secondary slots, which are assigned
+	/// in round-robin manner.
+	pub secondary_slots: bool,
+}
+
+#[cfg(feature = "std")]
+impl slots::SlotData for BabeConfiguration {
+	fn slot_duration(&self) -> u64 {
+		self.slot_duration
+	}
+
+	const SLOT_KEY: &'static [u8] = b"babe_configuration";
+}
+
+sr_api::decl_runtime_apis! {
+	/// API necessary for block authorship with BABE.
+	pub trait BabeApi {
+		/// Return the configuration for BABE. Currently,
+		/// only the value provided by this type at genesis will be used.
+		///
+		/// Dynamic configuration may be supported in the future.
+		fn configuration() -> BabeConfiguration;
+	}
+}

substrate/primitives/consensus/common/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "substrate-consensus-common"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Common utilities for substrate consensus"
+edition = "2018"
+
+[dependencies]
+derive_more = "0.15.0"
+libp2p = { version = "0.13.0", default-features = false }
+log = "0.4.8"
+primitives = { package = "substrate-primitives", path= "../../core" }
+inherents = { package = "substrate-inherents", path = "../../inherents" }
+futures-preview = "0.3.0-alpha.19"
+futures-timer = "0.4.0"
+rstd = { package = "sr-std", path = "../../sr-std" }
+runtime_version = { package = "sr-version", path = "../../sr-version" }
+sr-primitives = {  path = "../../sr-primitives" }
+codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] }
+parking_lot = "0.9.0"
+
+[dev-dependencies]
+test-client = { package = "substrate-test-runtime-client", path = "../../../test/utils/runtime/client" }
+
+[features]
+default = []

substrate/primitives/consensus/common/src/block_import.rs

@@ -0,0 +1,284 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Block import helpers.
+
+use sr_primitives::traits::{Block as BlockT, DigestItemFor, Header as HeaderT, NumberFor};
+use sr_primitives::Justification;
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use crate::import_queue::{Verifier, CacheKeyId};
+
+/// Block import result.
+#[derive(Debug, PartialEq, Eq)]
+pub enum ImportResult {
+	/// Block imported.
+	Imported(ImportedAux),
+	/// Already in the blockchain.
+	AlreadyInChain,
+	/// Block or parent is known to be bad.
+	KnownBad,
+	/// Block parent is not in the chain.
+	UnknownParent,
+	/// Parent state is missing.
+	MissingState,
+}
+
+/// Auxiliary data associated with an imported block result.
+#[derive(Debug, Default, PartialEq, Eq)]
+pub struct ImportedAux {
+	/// Only the header has been imported. Block body verification was skipped.
+	pub header_only: bool,
+	/// Clear all pending justification requests.
+	pub clear_justification_requests: bool,
+	/// Request a justification for the given block.
+	pub needs_justification: bool,
+	/// Received a bad justification.
+	pub bad_justification: bool,
+	/// Request a finality proof for the given block.
+	pub needs_finality_proof: bool,
+	/// Whether the block that was imported is the new best block.
+	pub is_new_best: bool,
+}
+
+impl ImportResult {
+	/// Returns default value for `ImportResult::Imported` with
+	/// `clear_justification_requests`, `needs_justification`,
+	/// `bad_justification` and `needs_finality_proof` set to false.
+	pub fn imported(is_new_best: bool) -> ImportResult {
+		let mut aux = ImportedAux::default();
+		aux.is_new_best = is_new_best;
+
+		ImportResult::Imported(aux)
+	}
+}
+
+/// Block data origin.
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum BlockOrigin {
+	/// Genesis block built into the client.
+	Genesis,
+	/// Block is part of the initial sync with the network.
+	NetworkInitialSync,
+	/// Block was broadcasted on the network.
+	NetworkBroadcast,
+	/// Block that was received from the network and validated in the consensus process.
+	ConsensusBroadcast,
+	/// Block that was collated by this node.
+	Own,
+	/// Block was imported from a file.
+	File,
+}
+
+/// Fork choice strategy.
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum ForkChoiceStrategy {
+	/// Longest chain fork choice.
+	LongestChain,
+	/// Custom fork choice rule, where true indicates the new block should be the best block.
+	Custom(bool),
+}
+
+/// Data required to check validity of a Block.
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub struct BlockCheckParams<Block: BlockT> {
+	/// Hash of the block that we verify.
+	pub hash: Block::Hash,
+	/// Block number of the block that we verify.
+	pub number: NumberFor<Block>,
+	/// Parent hash of the block that we verify.
+	pub parent_hash: Block::Hash,
+	/// Allow importing the block skipping state verification if parent state is missing.
+	pub allow_missing_state: bool,
+}
+
+/// Data required to import a Block.
+pub struct BlockImportParams<Block: BlockT> {
+	/// Origin of the Block
+	pub origin: BlockOrigin,
+	/// The header, without consensus post-digests applied. This should be in the same
+	/// state as it comes out of the runtime.
+	///
+	/// Consensus engines which alter the header (by adding post-runtime digests)
+	/// should strip those off in the initial verification process and pass them
+	/// via the `post_digests` field. During block authorship, they should
+	/// not be pushed to the header directly.
+	///
+	/// The reason for this distinction is so the header can be directly
+	/// re-executed in a runtime that checks digest equivalence -- the
+	/// post-runtime digests are pushed back on after.
+	pub header: Block::Header,
+	/// Justification provided for this block from the outside.
+	pub justification: Option<Justification>,
+	/// Digest items that have been added after the runtime for external
+	/// work, like a consensus signature.
+	pub post_digests: Vec<DigestItemFor<Block>>,
+	/// Block's body
+	pub body: Option<Vec<Block::Extrinsic>>,
+	/// Is this block finalized already?
+	/// `true` implies instant finality.
+	pub finalized: bool,
+	/// Auxiliary consensus data produced by the block.
+	/// Contains a list of key-value pairs. If values are `None`, the keys
+	/// will be deleted.
+	pub auxiliary: Vec<(Vec<u8>, Option<Vec<u8>>)>,
+	/// Fork choice strategy of this import. This should only be set by a
+	/// synchronous import, otherwise it may race against other imports.
+	pub fork_choice: ForkChoiceStrategy,
+	/// Allow importing the block skipping state verification if parent state is missing.
+	pub allow_missing_state: bool,
+}
+
+impl<Block: BlockT> BlockImportParams<Block> {
+	/// Deconstruct the justified header into parts.
+	pub fn into_inner(self)
+		-> (
+			BlockOrigin,
+			<Block as BlockT>::Header,
+			Option<Justification>,
+			Vec<DigestItemFor<Block>>,
+			Option<Vec<<Block as BlockT>::Extrinsic>>,
+			bool,
+			Vec<(Vec<u8>, Option<Vec<u8>>)>,
+		) {
+		(
+			self.origin,
+			self.header,
+			self.justification,
+			self.post_digests,
+			self.body,
+			self.finalized,
+			self.auxiliary,
+		)
+	}
+
+	/// Get a handle to full header (with post-digests applied).
+	pub fn post_header(&self) -> Cow<Block::Header> {
+		if self.post_digests.is_empty() {
+			Cow::Borrowed(&self.header)
+		} else {
+			Cow::Owned({
+				let mut hdr = self.header.clone();
+				for digest_item in &self.post_digests {
+					hdr.digest_mut().push(digest_item.clone());
+				}
+
+				hdr
+			})
+		}
+	}
+}
+
+/// Block import trait.
+pub trait BlockImport<B: BlockT> {
+	type Error: ::std::error::Error + Send + 'static;
+
+	/// Check block preconditions.
+	fn check_block(
+		&mut self,
+		block: BlockCheckParams<B>,
+	) -> Result<ImportResult, Self::Error>;
+
+	/// Import a block.
+	///
+	/// Cached data can be accessed through the blockchain cache.
+	fn import_block(
+		&mut self,
+		block: BlockImportParams<B>,
+		cache: HashMap<CacheKeyId, Vec<u8>>,
+	) -> Result<ImportResult, Self::Error>;
+}
+
+impl<B: BlockT> BlockImport<B> for crate::import_queue::BoxBlockImport<B> {
+	type Error = crate::error::Error;
+
+	/// Check block preconditions.
+	fn check_block(
+		&mut self,
+		block: BlockCheckParams<B>,
+	) -> Result<ImportResult, Self::Error> {
+		(**self).check_block(block)
+	}
+
+	/// Import a block.
+	///
+	/// Cached data can be accessed through the blockchain cache.
+	fn import_block(
+		&mut self,
+		block: BlockImportParams<B>,
+		cache: HashMap<CacheKeyId, Vec<u8>>,
+	) -> Result<ImportResult, Self::Error> {
+		(**self).import_block(block, cache)
+	}
+}
+
+impl<B: BlockT, T, E: std::error::Error + Send + 'static> BlockImport<B> for Arc<T>
+where for<'r> &'r T: BlockImport<B, Error = E>
+{
+	type Error = E;
+
+	fn check_block(
+		&mut self,
+		block: BlockCheckParams<B>,
+	) -> Result<ImportResult, Self::Error> {
+		(&**self).check_block(block)
+	}
+
+	fn import_block(
+		&mut self,
+		block: BlockImportParams<B>,
+		cache: HashMap<CacheKeyId, Vec<u8>>,
+	) -> Result<ImportResult, Self::Error> {
+		(&**self).import_block(block, cache)
+	}
+}
+
+/// Justification import trait
+pub trait JustificationImport<B: BlockT> {
+	type Error: ::std::error::Error + Send + 'static;
+
+	/// Called by the import queue when it is started. Returns a list of justifications to request
+	/// from the network.
+	fn on_start(&mut self) -> Vec<(B::Hash, NumberFor<B>)> { Vec::new() }
+
+	/// Import a Block justification and finalize the given block.
+	fn import_justification(
+		&mut self,
+		hash: B::Hash,
+		number: NumberFor<B>,
+		justification: Justification,
+	) -> Result<(), Self::Error>;
+}
+
+/// Finality proof import trait.
+pub trait FinalityProofImport<B: BlockT> {
+	type Error: std::error::Error + Send + 'static;
+
+	/// Called by the import queue when it is started. Returns a list of finality proofs to request
+	/// from the network.
+	fn on_start(&mut self) -> Vec<(B::Hash, NumberFor<B>)> { Vec::new() }
+
+	/// Import a Block justification and finalize the given block. Returns finalized block or error.
+	fn import_finality_proof(
+		&mut self,
+		hash: B::Hash,
+		number: NumberFor<B>,
+		finality_proof: Vec<u8>,
+		verifier: &mut dyn Verifier<B>,
+	) -> Result<(B::Hash, NumberFor<B>), Self::Error>;
+}

substrate/primitives/consensus/common/src/block_validation.rs

@@ -0,0 +1,66 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+//
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Block announcement validation.
+
+use crate::BlockStatus;
+use sr_primitives::{generic::BlockId, traits::Block};
+use std::{error::Error, sync::Arc};
+
+/// A type which provides access to chain information.
+pub trait Chain<B: Block> {
+	/// Retrieve the status of the block denoted by the given [`BlockId`].
+	fn block_status(&self, id: &BlockId<B>) -> Result<BlockStatus, Box<dyn Error + Send>>;
+}
+
+impl<T: Chain<B>, B: Block> Chain<B> for Arc<T> {
+	fn block_status(&self, id: &BlockId<B>) -> Result<BlockStatus, Box<dyn Error + Send>> {
+		(&**self).block_status(id)
+	}
+}
+
+/// Result of `BlockAnnounceValidator::validate`.
+#[derive(Debug, PartialEq, Eq)]
+pub enum Validation {
+	/// Valid block announcement.
+	Success,
+	/// Invalid block announcement.
+	Failure,
+}
+
+/// Type which checks incoming block announcements.
+pub trait BlockAnnounceValidator<B: Block> {
+	/// Validate the announced header and its associated data.
+	fn validate(&mut self, header: &B::Header, data: &[u8]) -> Result<Validation, Box<dyn Error + Send>>;
+}
+
+/// Default implementation of `BlockAnnounceValidator`.
+#[derive(Debug)]
+pub struct DefaultBlockAnnounceValidator<C> {
+	chain: C
+}
+
+impl<C> DefaultBlockAnnounceValidator<C> {
+	pub fn new(chain: C) -> Self {
+		Self { chain }
+	}
+}
+
+impl<B: Block, C: Chain<B>> BlockAnnounceValidator<B> for DefaultBlockAnnounceValidator<C> {
+	fn validate(&mut self, _h: &B::Header, _d: &[u8]) -> Result<Validation, Box<dyn Error + Send>> {
+		Ok(Validation::Success)
+	}
+}

substrate/primitives/consensus/common/src/error.rs

@@ -0,0 +1,84 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Error types in Consensus
+use runtime_version::RuntimeVersion;
+use primitives::ed25519::{Public, Signature};
+use std::error;
+
+/// Result type alias.
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// Error type.
+#[derive(Debug, derive_more::Display, derive_more::From)]
+pub enum Error {
+	/// Missing state at block with given descriptor.
+	#[display(fmt="State unavailable at block {}", _0)]
+	StateUnavailable(String),
+	/// I/O terminated unexpectedly
+	#[display(fmt="I/O terminated unexpectedly.")]
+	IoTerminated,
+	/// Unable to schedule wakeup.
+	#[display(fmt="Timer error: {}", _0)]
+	FaultyTimer(std::io::Error),
+	/// Error while working with inherent data.
+	#[display(fmt="InherentData error: {}", _0)]
+	InherentData(inherents::Error),
+	/// Unable to propose a block.
+	#[display(fmt="Unable to create block proposal.")]
+	CannotPropose,
+	/// Error checking signature
+	#[display(fmt="Message signature {:?} by {:?} is invalid.", _0, _1)]
+	InvalidSignature(Signature, Public),
+	/// Invalid authorities set received from the runtime.
+	#[display(fmt="Current state of blockchain has invalid authorities set")]
+	InvalidAuthoritiesSet,
+	/// Account is not an authority.
+	#[display(fmt="Message sender {:?} is not a valid authority.", _0)]
+	InvalidAuthority(Public),
+	/// Authoring interface does not match the runtime.
+	#[display(fmt="Authoring for current \
+				runtime is not supported. Native ({}) cannot author for on-chain ({}).", native, on_chain)]
+	IncompatibleAuthoringRuntime { native: RuntimeVersion, on_chain: RuntimeVersion },
+	/// Authoring interface does not match the runtime.
+	#[display(fmt="Authoring for current runtime is not supported since it has no version.")]
+	RuntimeVersionMissing,
+	/// Authoring interface does not match the runtime.
+	#[display(fmt="Authoring in current build is not supported since it has no runtime.")]
+	NativeRuntimeMissing,
+	/// Justification requirements not met.
+	#[display(fmt="Invalid justification.")]
+	InvalidJustification,
+	/// Some other error.
+	#[display(fmt="Other error: {}", _0)]
+	Other(Box<dyn error::Error + Send>),
+	/// Error from the client while importing
+	#[display(fmt="Import failed: {}", _0)]
+	ClientImport(String),
+	/// Error from the client while importing
+	#[display(fmt="Chain lookup failed: {}", _0)]
+	ChainLookup(String),
+}
+
+impl error::Error for Error {
+	fn source(&self) -> Option<&(dyn error::Error + 'static)> {
+		match self {
+			Error::FaultyTimer(ref err) => Some(err),
+			Error::Other(ref err) => Some(&**err),
+			_ => None,
+		}
+	}
+}

substrate/primitives/consensus/common/src/evaluation.rs

@@ -0,0 +1,84 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Block evaluation and evaluation errors.
+
+use super::MAX_BLOCK_SIZE;
+
+use codec::Encode;
+use sr_primitives::traits::{Block as BlockT, Header as HeaderT, One, CheckedConversion};
+
+// This is just a best effort to encode the number. None indicated that it's too big to encode
+// in a u128.
+type BlockNumber = Option<u128>;
+
+/// Result type alias.
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// Error type.
+#[derive(Debug, derive_more::Display)]
+pub enum Error {
+	/// Proposal provided not a block.
+	#[display(fmt="Proposal provided not a block: decoding error: {}", _0)]
+	BadProposalFormat(codec::Error),
+	/// Proposal had wrong parent hash.
+	#[display(fmt="Proposal had wrong parent hash. Expected {:?}, got {:?}", expected, got)]
+	WrongParentHash { expected: String, got: String },
+	/// Proposal had wrong number.
+	#[display(fmt="Proposal had wrong number. Expected {:?}, got {:?}", expected, got)]
+	WrongNumber { expected: BlockNumber, got: BlockNumber },
+	/// Proposal exceeded the maximum size.
+	#[display(
+		fmt="Proposal exceeded the maximum size of {} by {} bytes.",
+		"MAX_BLOCK_SIZE", "_0.saturating_sub(MAX_BLOCK_SIZE)"
+	)]
+	ProposalTooLarge(usize),
+}
+
+impl std::error::Error for Error {}
+
+/// Attempt to evaluate a substrate block as a node block, returning error
+/// upon any initial validity checks failing.
+pub fn evaluate_initial<Block: BlockT>(
+	proposal: &Block,
+	parent_hash: &<Block as BlockT>::Hash,
+	parent_number: <<Block as BlockT>::Header as HeaderT>::Number,
+) -> Result<()> {
+
+	let encoded = Encode::encode(proposal);
+	let proposal = Block::decode(&mut &encoded[..])
+		.map_err(|e| Error::BadProposalFormat(e))?;
+
+	if encoded.len() > MAX_BLOCK_SIZE {
+		return Err(Error::ProposalTooLarge(encoded.len()))
+	}
+
+	if *parent_hash != *proposal.header().parent_hash() {
+		return Err(Error::WrongParentHash {
+			expected: format!("{:?}", *parent_hash),
+			got: format!("{:?}", proposal.header().parent_hash())
+		});
+	}
+
+	if parent_number + One::one() != *proposal.header().number() {
+		return Err(Error::WrongNumber {
+			expected: parent_number.checked_into::<u128>().map(|x| x + 1),
+			got: (*proposal.header().number()).checked_into::<u128>(),
+		});
+	}
+
+	Ok(())
+}

substrate/primitives/consensus/common/src/import_queue.rs

@@ -0,0 +1,255 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Import Queue primitive: something which can verify and import blocks.
+//!
+//! This serves as an intermediate and abstracted step between synchronization
+//! and import. Each mode of consensus will have its own requirements for block
+//! verification. Some algorithms can verify in parallel, while others only
+//! sequentially.
+//!
+//! The `ImportQueue` trait allows such verification strategies to be
+//! instantiated. The `BasicQueue` and `BasicVerifier` traits allow serial
+//! queues to be instantiated simply.
+
+use std::collections::HashMap;
+use sr_primitives::{Justification, traits::{Block as BlockT, Header as _, NumberFor}};
+use crate::error::Error as ConsensusError;
+use crate::block_import::{
+	BlockImport, BlockOrigin, BlockImportParams, ImportedAux, JustificationImport, ImportResult,
+	BlockCheckParams, FinalityProofImport,
+};
+
+pub use basic_queue::BasicQueue;
+
+mod basic_queue;
+pub mod buffered_link;
+
+/// Shared block import struct used by the queue.
+pub type BoxBlockImport<B> = Box<dyn BlockImport<B, Error = ConsensusError> + Send + Sync>;
+
+/// Shared justification import struct used by the queue.
+pub type BoxJustificationImport<B> = Box<dyn JustificationImport<B, Error=ConsensusError> + Send + Sync>;
+
+/// Shared finality proof import struct used by the queue.
+pub type BoxFinalityProofImport<B> = Box<dyn FinalityProofImport<B, Error=ConsensusError> + Send + Sync>;
+
+/// Maps to the Origin used by the network.
+pub type Origin = libp2p::PeerId;
+
+/// Block data used by the queue.
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub struct IncomingBlock<B: BlockT> {
+	/// Block header hash.
+	pub hash: <B as BlockT>::Hash,
+	/// Block header if requested.
+	pub header: Option<<B as BlockT>::Header>,
+	/// Block body if requested.
+	pub body: Option<Vec<<B as BlockT>::Extrinsic>>,
+	/// Justification if requested.
+	pub justification: Option<Justification>,
+	/// The peer, we received this from
+	pub origin: Option<Origin>,
+	/// Allow importing the block skipping state verification if parent state is missing.
+	pub allow_missing_state: bool,
+}
+
+/// Type of keys in the blockchain cache that consensus module could use for its needs.
+pub type CacheKeyId = [u8; 4];
+
+/// Verify a justification of a block
+pub trait Verifier<B: BlockT>: Send + Sync {
+	/// Verify the given data and return the BlockImportParams and an optional
+	/// new set of validators to import. If not, err with an Error-Message
+	/// presented to the User in the logs.
+	fn verify(
+		&mut self,
+		origin: BlockOrigin,
+		header: B::Header,
+		justification: Option<Justification>,
+		body: Option<Vec<B::Extrinsic>>,
+	) -> Result<(BlockImportParams<B>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String>;
+}
+
+/// Blocks import queue API.
+///
+/// The `import_*` methods can be called in order to send elements for the import queue to verify.
+/// Afterwards, call `poll_actions` to determine how to respond to these elements.
+pub trait ImportQueue<B: BlockT>: Send {
+	/// Import bunch of blocks.
+	fn import_blocks(&mut self, origin: BlockOrigin, blocks: Vec<IncomingBlock<B>>);
+	/// Import a block justification.
+	fn import_justification(
+		&mut self,
+		who: Origin,
+		hash: B::Hash,
+		number: NumberFor<B>,
+		justification: Justification
+	);
+	/// Import block finality proof.
+	fn import_finality_proof(
+		&mut self,
+		who: Origin,
+		hash: B::Hash,
+		number: NumberFor<B>,
+		finality_proof: Vec<u8>
+	);
+
+	/// Polls for actions to perform on the network.
+	///
+	/// This method should behave in a way similar to `Future::poll`. It can register the current
+	/// task and notify later when more actions are ready to be polled. To continue the comparison,
+	/// it is as if this method always returned `Poll::Pending`.
+	fn poll_actions(&mut self, cx: &mut futures::task::Context, link: &mut dyn Link<B>);
+}
+
+/// Hooks that the verification queue can use to influence the synchronization
+/// algorithm.
+pub trait Link<B: BlockT>: Send {
+	/// Batch of blocks imported, with or without error.
+	fn blocks_processed(
+		&mut self,
+		_imported: usize,
+		_count: usize,
+		_results: Vec<(Result<BlockImportResult<NumberFor<B>>, BlockImportError>, B::Hash)>
+	) {}
+	/// Justification import result.
+	fn justification_imported(&mut self, _who: Origin, _hash: &B::Hash, _number: NumberFor<B>, _success: bool) {}
+	/// Request a justification for the given block.
+	fn request_justification(&mut self, _hash: &B::Hash, _number: NumberFor<B>) {}
+	/// Finality proof import result.
+	///
+	/// Even though we have asked for finality proof of block A, provider could return proof of
+	/// some earlier block B, if the proof for A was too large. The sync module should continue
+	/// asking for proof of A in this case.
+	fn finality_proof_imported(
+		&mut self,
+		_who: Origin,
+		_request_block: (B::Hash, NumberFor<B>),
+		_finalization_result: Result<(B::Hash, NumberFor<B>), ()>,
+	) {}
+	/// Request a finality proof for the given block.
+	fn request_finality_proof(&mut self, _hash: &B::Hash, _number: NumberFor<B>) {}
+}
+
+/// Block import successful result.
+#[derive(Debug, PartialEq)]
+pub enum BlockImportResult<N: ::std::fmt::Debug + PartialEq> {
+	/// Imported known block.
+	ImportedKnown(N),
+	/// Imported unknown block.
+	ImportedUnknown(N, ImportedAux, Option<Origin>),
+}
+
+/// Block import error.
+#[derive(Debug)]
+pub enum BlockImportError {
+	/// Block missed header, can't be imported
+	IncompleteHeader(Option<Origin>),
+	/// Block verification failed, can't be imported
+	VerificationFailed(Option<Origin>, String),
+	/// Block is known to be Bad
+	BadBlock(Option<Origin>),
+	/// Parent state is missing.
+	MissingState,
+	/// Block has an unknown parent
+	UnknownParent,
+	/// Block import has been cancelled. This can happen if the parent block fails to be imported.
+	Cancelled,
+	/// Other error.
+	Other(ConsensusError),
+}
+
+/// Single block import function.
+pub fn import_single_block<B: BlockT, V: Verifier<B>>(
+	import_handle: &mut dyn BlockImport<B, Error = ConsensusError>,
+	block_origin: BlockOrigin,
+	block: IncomingBlock<B>,
+	verifier: &mut V,
+) -> Result<BlockImportResult<NumberFor<B>>, BlockImportError> {
+	let peer = block.origin;
+
+	let (header, justification) = match (block.header, block.justification) {
+		(Some(header), justification) => (header, justification),
+		(None, _) => {
+			if let Some(ref peer) = peer {
+				debug!(target: "sync", "Header {} was not provided by {} ", block.hash, peer);
+			} else {
+				debug!(target: "sync", "Header {} was not provided ", block.hash);
+			}
+			return Err(BlockImportError::IncompleteHeader(peer))
+		},
+	};
+
+	trace!(target: "sync", "Header {} has {:?} logs", block.hash, header.digest().logs().len());
+
+	let number = header.number().clone();
+	let hash = header.hash();
+	let parent_hash = header.parent_hash().clone();
+
+	let import_error = |e| {
+		match e {
+			Ok(ImportResult::AlreadyInChain) => {
+				trace!(target: "sync", "Block already in chain {}: {:?}", number, hash);
+				Ok(BlockImportResult::ImportedKnown(number))
+			},
+			Ok(ImportResult::Imported(aux)) => Ok(BlockImportResult::ImportedUnknown(number, aux, peer.clone())),
+			Ok(ImportResult::MissingState) => {
+				debug!(target: "sync", "Parent state is missing for {}: {:?}, parent: {:?}", number, hash, parent_hash);
+				Err(BlockImportError::MissingState)
+			},
+			Ok(ImportResult::UnknownParent) => {
+				debug!(target: "sync", "Block with unknown parent {}: {:?}, parent: {:?}", number, hash, parent_hash);
+				Err(BlockImportError::UnknownParent)
+			},
+			Ok(ImportResult::KnownBad) => {
+				debug!(target: "sync", "Peer gave us a bad block {}: {:?}", number, hash);
+				Err(BlockImportError::BadBlock(peer.clone()))
+			},
+			Err(e) => {
+				debug!(target: "sync", "Error importing block {}: {:?}: {:?}", number, hash, e);
+				Err(BlockImportError::Other(e))
+			}
+		}
+	};
+	match import_error(import_handle.check_block(BlockCheckParams {
+		hash,
+		number,
+		parent_hash,
+		allow_missing_state: block.allow_missing_state,
+	}))? {
+		BlockImportResult::ImportedUnknown { .. } => (),
+		r => return Ok(r), // Any other successful result means that the block is already imported.
+	}
+
+	let (mut import_block, maybe_keys) = verifier.verify(block_origin, header, justification, block.body)
+		.map_err(|msg| {
+			if let Some(ref peer) = peer {
+				trace!(target: "sync", "Verifying {}({}) from {} failed: {}", number, hash, peer, msg);
+			} else {
+				trace!(target: "sync", "Verifying {}({}) failed: {}", number, hash, msg);
+			}
+			BlockImportError::VerificationFailed(peer.clone(), msg)
+		})?;
+
+	let mut cache = HashMap::new();
+	if let Some(keys) = maybe_keys {
+		cache.extend(keys.into_iter());
+	}
+	import_block.allow_missing_state = block.allow_missing_state;
+
+	import_error(import_handle.import_block(import_block, cache))
+}

substrate/primitives/consensus/common/src/import_queue/basic_queue.rs

@@ -0,0 +1,408 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{mem, pin::Pin, time::Duration};
+use futures::{prelude::*, channel::mpsc, task::Context, task::Poll};
+use futures_timer::Delay;
+use sr_primitives::{Justification, traits::{Block as BlockT, Header as HeaderT, NumberFor}};
+
+use crate::block_import::BlockOrigin;
+use crate::import_queue::{
+	BlockImportResult, BlockImportError, Verifier, BoxBlockImport, BoxFinalityProofImport,
+	BoxJustificationImport, ImportQueue, Link, Origin,
+	IncomingBlock, import_single_block,
+	buffered_link::{self, BufferedLinkSender, BufferedLinkReceiver}
+};
+
+/// Interface to a basic block import queue that is importing blocks sequentially in a separate
+/// task, with pluggable verification.
+pub struct BasicQueue<B: BlockT> {
+	/// Channel to send messages to the background task.
+	sender: mpsc::UnboundedSender<ToWorkerMsg<B>>,
+	/// Results coming from the worker task.
+	result_port: BufferedLinkReceiver<B>,
+	/// If it isn't possible to spawn the future in `future_to_spawn` (which is notably the case in
+	/// "no std" environment), we instead put it in `manual_poll`. It is then polled manually from
+	/// `poll_actions`.
+	manual_poll: Option<Pin<Box<dyn Future<Output = ()> + Send>>>,
+	/// A thread pool where the background worker is being run.
+	pool: Option<futures::executor::ThreadPool>,
+}
+
+impl<B: BlockT> BasicQueue<B> {
+	/// Instantiate a new basic queue, with given verifier.
+	///
+	/// This creates a background task, and calls `on_start` on the justification importer and
+	/// finality proof importer.
+	pub fn new<V: 'static + Verifier<B>>(
+		verifier: V,
+		block_import: BoxBlockImport<B>,
+		justification_import: Option<BoxJustificationImport<B>>,
+		finality_proof_import: Option<BoxFinalityProofImport<B>>,
+	) -> Self {
+		let (result_sender, result_port) = buffered_link::buffered_link();
+		let (future, worker_sender) = BlockImportWorker::new(
+			result_sender,
+			verifier,
+			block_import,
+			justification_import,
+			finality_proof_import,
+		);
+
+		let mut pool = futures::executor::ThreadPool::builder()
+			.name_prefix("import-queue-worker-")
+			.pool_size(1)
+			.create()
+			.ok();
+
+		let manual_poll;
+		if let Some(pool) = &mut pool {
+			pool.spawn_ok(future);
+			manual_poll = None;
+		} else {
+			manual_poll = Some(Box::pin(future) as Pin<Box<_>>);
+		}
+
+		Self {
+			sender: worker_sender,
+			result_port,
+			manual_poll,
+			pool,
+		}
+	}
+}
+
+impl<B: BlockT> ImportQueue<B> for BasicQueue<B> {
+	fn import_blocks(&mut self, origin: BlockOrigin, blocks: Vec<IncomingBlock<B>>) {
+		if blocks.is_empty() {
+			return;
+		}
+
+		trace!(target: "sync", "Scheduling {} blocks for import", blocks.len());
+		let _ = self.sender.unbounded_send(ToWorkerMsg::ImportBlocks(origin, blocks));
+	}
+
+	fn import_justification(
+		&mut self,
+		who: Origin,
+		hash: B::Hash,
+		number: NumberFor<B>,
+		justification: Justification
+	) {
+		let _ = self.sender.unbounded_send(ToWorkerMsg::ImportJustification(who.clone(), hash, number, justification));
+	}
+
+	fn import_finality_proof(&mut self, who: Origin, hash: B::Hash, number: NumberFor<B>, finality_proof: Vec<u8>) {
+		trace!(target: "sync", "Scheduling finality proof of {}/{} for import", number, hash);
+		let _ = self.sender.unbounded_send(ToWorkerMsg::ImportFinalityProof(who, hash, number, finality_proof));
+	}
+
+	fn poll_actions(&mut self, cx: &mut Context, link: &mut dyn Link<B>) {
+		// As a backup mechanism, if we failed to spawn the `future_to_spawn`, we instead poll
+		// manually here.
+		if let Some(manual_poll) = self.manual_poll.as_mut() {
+			match Future::poll(Pin::new(manual_poll), cx) {
+				Poll::Pending => {}
+				_ => self.manual_poll = None,
+			}
+		}
+
+		self.result_port.poll_actions(cx, link);
+	}
+}
+
+/// Message destinated to the background worker.
+#[derive(Debug)]
+enum ToWorkerMsg<B: BlockT> {
+	ImportBlocks(BlockOrigin, Vec<IncomingBlock<B>>),
+	ImportJustification(Origin, B::Hash, NumberFor<B>, Justification),
+	ImportFinalityProof(Origin, B::Hash, NumberFor<B>, Vec<u8>),
+}
+
+struct BlockImportWorker<B: BlockT> {
+	result_sender: BufferedLinkSender<B>,
+	justification_import: Option<BoxJustificationImport<B>>,
+	finality_proof_import: Option<BoxFinalityProofImport<B>>,
+	delay_between_blocks: Duration,
+}
+
+impl<B: BlockT> BlockImportWorker<B> {
+	fn new<V: 'static + Verifier<B>>(
+		result_sender: BufferedLinkSender<B>,
+		verifier: V,
+		block_import: BoxBlockImport<B>,
+		justification_import: Option<BoxJustificationImport<B>>,
+		finality_proof_import: Option<BoxFinalityProofImport<B>>,
+	) -> (impl Future<Output = ()> + Send, mpsc::UnboundedSender<ToWorkerMsg<B>>) {
+		let (sender, mut port) = mpsc::unbounded();
+
+		let mut worker = BlockImportWorker {
+			result_sender,
+			justification_import,
+			finality_proof_import,
+			delay_between_blocks: Duration::new(0, 0),
+		};
+
+		// Let's initialize `justification_import` and `finality_proof_import`.
+		if let Some(justification_import) = worker.justification_import.as_mut() {
+			for (hash, number) in justification_import.on_start() {
+				worker.result_sender.request_justification(&hash, number);
+			}
+		}
+		if let Some(finality_proof_import) = worker.finality_proof_import.as_mut() {
+			for (hash, number) in finality_proof_import.on_start() {
+				worker.result_sender.request_finality_proof(&hash, number);
+			}
+		}
+
+		// The future below has two possible states:
+		//
+		// - Currently importing many blocks, in which case `importing` is `Some` and contains a
+		//   `Future`, and `block_import` is `None`.
+		// - Something else, in which case `block_import` is `Some` and `importing` is None.
+		//
+		let mut block_import_verifier = Some((block_import, verifier));
+		let mut importing = None;
+
+		let future = futures::future::poll_fn(move |cx| {
+			loop {
+				// If the results sender is closed, that means that the import queue is shutting
+				// down and we should end this future.
+				if worker.result_sender.is_closed() {
+					return Poll::Ready(())
+				}
+
+				// If we are in the process of importing a bunch of block, let's resume this
+				// process before doing anything more.
+				if let Some(imp_fut) = importing.as_mut() {
+					match Future::poll(Pin::new(imp_fut), cx) {
+						Poll::Pending => return Poll::Pending,
+						Poll::Ready((bi, verif)) => {
+							block_import_verifier = Some((bi, verif));
+							importing = None;
+						},
+					}
+				}
+
+				debug_assert!(importing.is_none());
+				debug_assert!(block_import_verifier.is_some());
+
+				// Grab the next action request sent to the import queue.
+				let msg = match Stream::poll_next(Pin::new(&mut port), cx) {
+					Poll::Ready(Some(msg)) => msg,
+					Poll::Ready(None) => return Poll::Ready(()),
+					Poll::Pending => return Poll::Pending,
+				};
+
+				match msg {
+					ToWorkerMsg::ImportBlocks(origin, blocks) => {
+						// On blocks import request, we merely *start* the process and store
+						// a `Future` into `importing`.
+						let (bi, verif) = block_import_verifier.take()
+							.expect("block_import_verifier is always Some; qed");
+						importing = Some(worker.import_a_batch_of_blocks(bi, verif, origin, blocks));
+					},
+					ToWorkerMsg::ImportFinalityProof(who, hash, number, proof) => {
+						let (_, verif) = block_import_verifier.as_mut()
+							.expect("block_import_verifier is always Some; qed");
+						worker.import_finality_proof(verif, who, hash, number, proof);
+					},
+					ToWorkerMsg::ImportJustification(who, hash, number, justification) => {
+						worker.import_justification(who, hash, number, justification);
+					}
+				}
+			}
+		});
+
+		(future, sender)
+	}
+
+	/// Returns a `Future` that imports the given blocks and sends the results on
+	/// `self.result_sender`.
+	///
+	/// For lifetime reasons, the `BlockImport` implementation must be passed by value, and is
+	/// yielded back in the output once the import is finished.
+	fn import_a_batch_of_blocks<V: 'static + Verifier<B>>(
+		&mut self,
+		block_import: BoxBlockImport<B>,
+		verifier: V,
+		origin: BlockOrigin,
+		blocks: Vec<IncomingBlock<B>>
+	) -> impl Future<Output = (BoxBlockImport<B>, V)> {
+		let mut result_sender = self.result_sender.clone();
+
+		import_many_blocks(block_import, origin, blocks, verifier, self.delay_between_blocks)
+			.then(move |(imported, count, results, block_import, verifier)| {
+				result_sender.blocks_processed(imported, count, results);
+				future::ready((block_import, verifier))
+			})
+	}
+
+	fn import_finality_proof<V: 'static + Verifier<B>>(
+		&mut self,
+		verifier: &mut V,
+		who: Origin,
+		hash: B::Hash,
+		number: NumberFor<B>,
+		finality_proof: Vec<u8>
+	) {
+		let result = self.finality_proof_import.as_mut().map(|finality_proof_import| {
+			finality_proof_import.import_finality_proof(hash, number, finality_proof, verifier)
+				.map_err(|e| {
+					debug!(
+						"Finality proof import failed with {:?} for hash: {:?} number: {:?} coming from node: {:?}",
+						e,
+						hash,
+						number,
+						who,
+					);
+				})
+		}).unwrap_or(Err(()));
+
+		trace!(target: "sync", "Imported finality proof for {}/{}", number, hash);
+		self.result_sender.finality_proof_imported(who, (hash, number), result);
+	}
+
+	fn import_justification(
+		&mut self,
+		who: Origin,
+		hash: B::Hash,
+		number: NumberFor<B>,
+		justification: Justification
+	) {
+		let success = self.justification_import.as_mut().map(|justification_import| {
+			justification_import.import_justification(hash, number, justification)
+				.map_err(|e| {
+					debug!(
+						target: "sync",
+						"Justification import failed with {:?} for hash: {:?} number: {:?} coming from node: {:?}",
+						e,
+						hash,
+						number,
+						who,
+					);
+					e
+				}).is_ok()
+		}).unwrap_or(false);
+
+		self.result_sender.justification_imported(who, &hash, number, success);
+	}
+}
+
+/// Import several blocks at once, returning import result for each block.
+///
+/// For lifetime reasons, the `BlockImport` implementation must be passed by value, and is yielded
+/// back in the output once the import is finished.
+///
+/// The returned `Future` yields at every imported block, which makes the execution more
+/// fine-grained and making it possible to interrupt the process.
+fn import_many_blocks<B: BlockT, V: Verifier<B>>(
+	import_handle: BoxBlockImport<B>,
+	blocks_origin: BlockOrigin,
+	blocks: Vec<IncomingBlock<B>>,
+	verifier: V,
+	delay_between_blocks: Duration,
+) -> impl Future<Output = (usize, usize, Vec<(
+	Result<BlockImportResult<NumberFor<B>>, BlockImportError>,
+	B::Hash,
+)>, BoxBlockImport<B>, V)> {
+	let count = blocks.len();
+
+	let blocks_range = match (
+		blocks.first().and_then(|b| b.header.as_ref().map(|h| h.number())),
+		blocks.last().and_then(|b| b.header.as_ref().map(|h| h.number())),
+	) {
+		(Some(first), Some(last)) if first != last => format!(" ({}..{})", first, last),
+		(Some(first), Some(_)) => format!(" ({})", first),
+		_ => Default::default(),
+	};
+
+	trace!(target: "sync", "Starting import of {} blocks {}", count, blocks_range);
+
+	let mut imported = 0;
+	let mut results = vec![];
+	let mut has_error = false;
+	let mut blocks = blocks.into_iter();
+	let mut import_handle = Some(import_handle);
+	let mut waiting = None;
+	let mut verifier = Some(verifier);
+
+	// Blocks in the response/drain should be in ascending order.
+
+	future::poll_fn(move |cx| {
+		// Handle the optional timer that makes us wait before the next import.
+		if let Some(waiting) = &mut waiting {
+			match Future::poll(Pin::new(waiting), cx) {
+				Poll::Ready(_) => {},
+				Poll::Pending => return Poll::Pending,
+			}
+		}
+		waiting = None;
+
+		// Is there any block left to import?
+		let block = match blocks.next() {
+			Some(b) => b,
+			None => {
+				// No block left to import, success!
+				let import_handle = import_handle.take()
+					.expect("Future polled again after it has finished");
+				let verifier = verifier.take()
+					.expect("Future polled again after it has finished");
+				let results = mem::replace(&mut results, Vec::new());
+				return Poll::Ready((imported, count, results, import_handle, verifier));
+			},
+		};
+
+		// We extract the content of `import_handle` and `verifier` only when the future ends,
+		// therefore `import_handle` and `verifier` are always `Some` here. It is illegal to poll
+		// a `Future` again after it has ended.
+		let import_handle = import_handle.as_mut()
+			.expect("Future polled again after it has finished");
+		let verifier = verifier.as_mut()
+			.expect("Future polled again after it has finished");
+
+		let block_number = block.header.as_ref().map(|h| h.number().clone());
+		let block_hash = block.hash;
+		let import_result = if has_error {
+			Err(BlockImportError::Cancelled)
+		} else {
+			// The actual import.
+			import_single_block(
+				&mut **import_handle,
+				blocks_origin.clone(),
+				block,
+				verifier,
+			)
+		};
+
+		if import_result.is_ok() {
+			trace!(target: "sync", "Block imported successfully {:?} ({})", block_number, block_hash);
+			imported += 1;
+		} else {
+			has_error = true;
+		}
+
+		results.push((import_result, block_hash));
+
+		// Notifies the current task again so that we re-execute this closure again for the next
+		// block.
+		if delay_between_blocks != Duration::new(0, 0) {
+			waiting = Some(Delay::new(delay_between_blocks));
+		}
+		cx.waker().wake_by_ref();
+		Poll::Pending
+	})
+}

substrate/primitives/consensus/common/src/import_queue/buffered_link.rs

@@ -0,0 +1,173 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Provides the `buffered_link` utility.
+//!
+//! The buffered link is a channel that allows buffering the method calls on `Link`.
+//!
+//! # Example
+//!
+//! ```
+//! use substrate_consensus_common::import_queue::Link;
+//! # use substrate_consensus_common::import_queue::buffered_link::buffered_link;
+//! # use test_client::runtime::Block;
+//! # struct DummyLink; impl Link<Block> for DummyLink {}
+//! # let mut my_link = DummyLink;
+//! let (mut tx, mut rx) = buffered_link::<Block>();
+//! tx.blocks_processed(0, 0, vec![]);
+//!
+//! // Calls `my_link.blocks_processed(0, 0, vec![])` when polled.
+//! let _fut = futures::future::poll_fn(move |cx| {
+//! 	rx.poll_actions(cx, &mut my_link);
+//! 	std::task::Poll::Pending::<()>
+//! });
+//! ```
+//!
+
+use futures::{prelude::*, channel::mpsc};
+use sr_primitives::traits::{Block as BlockT, NumberFor};
+use std::{pin::Pin, task::Context, task::Poll};
+use crate::import_queue::{Origin, Link, BlockImportResult, BlockImportError};
+
+/// Wraps around an unbounded channel from the `futures` crate. The sender implements `Link` and
+/// can be used to buffer commands, and the receiver can be used to poll said commands and transfer
+/// them to another link.
+pub fn buffered_link<B: BlockT>() -> (BufferedLinkSender<B>, BufferedLinkReceiver<B>) {
+	let (tx, rx) = mpsc::unbounded();
+	let tx = BufferedLinkSender { tx };
+	let rx = BufferedLinkReceiver { rx };
+	(tx, rx)
+}
+
+/// See [`buffered_link`].
+pub struct BufferedLinkSender<B: BlockT> {
+	tx: mpsc::UnboundedSender<BlockImportWorkerMsg<B>>,
+}
+
+impl<B: BlockT> BufferedLinkSender<B> {
+	/// Returns true if the sender points to nowhere.
+	///
+	/// Once `true` is returned, it is pointless to use the sender anymore.
+	pub fn is_closed(&self) -> bool {
+		self.tx.is_closed()
+	}
+}
+
+impl<B: BlockT> Clone for BufferedLinkSender<B> {
+	fn clone(&self) -> Self {
+		BufferedLinkSender {
+			tx: self.tx.clone(),
+		}
+	}
+}
+
+/// Internal buffered message.
+enum BlockImportWorkerMsg<B: BlockT> {
+	BlocksProcessed(usize, usize, Vec<(Result<BlockImportResult<NumberFor<B>>, BlockImportError>, B::Hash)>),
+	JustificationImported(Origin, B::Hash, NumberFor<B>, bool),
+	RequestJustification(B::Hash, NumberFor<B>),
+	FinalityProofImported(Origin, (B::Hash, NumberFor<B>), Result<(B::Hash, NumberFor<B>), ()>),
+	RequestFinalityProof(B::Hash, NumberFor<B>),
+}
+
+impl<B: BlockT> Link<B> for BufferedLinkSender<B> {
+	fn blocks_processed(
+		&mut self,
+		imported: usize,
+		count: usize,
+		results: Vec<(Result<BlockImportResult<NumberFor<B>>, BlockImportError>, B::Hash)>
+	) {
+		let _ = self.tx.unbounded_send(BlockImportWorkerMsg::BlocksProcessed(imported, count, results));
+	}
+
+	fn justification_imported(
+		&mut self,
+		who: Origin,
+		hash: &B::Hash,
+		number: NumberFor<B>,
+		success: bool
+	) {
+		let msg = BlockImportWorkerMsg::JustificationImported(who, hash.clone(), number, success);
+		let _ = self.tx.unbounded_send(msg);
+	}
+
+	fn request_justification(&mut self, hash: &B::Hash, number: NumberFor<B>) {
+		let _ = self.tx.unbounded_send(BlockImportWorkerMsg::RequestJustification(hash.clone(), number));
+	}
+
+	fn finality_proof_imported(
+		&mut self,
+		who: Origin,
+		request_block: (B::Hash, NumberFor<B>),
+		finalization_result: Result<(B::Hash, NumberFor<B>), ()>,
+	) {
+		let msg = BlockImportWorkerMsg::FinalityProofImported(who, request_block, finalization_result);
+		let _ = self.tx.unbounded_send(msg);
+	}
+
+	fn request_finality_proof(&mut self, hash: &B::Hash, number: NumberFor<B>) {
+		let _ = self.tx.unbounded_send(BlockImportWorkerMsg::RequestFinalityProof(hash.clone(), number));
+	}
+}
+
+/// See [`buffered_link`].
+pub struct BufferedLinkReceiver<B: BlockT> {
+	rx: mpsc::UnboundedReceiver<BlockImportWorkerMsg<B>>,
+}
+
+impl<B: BlockT> BufferedLinkReceiver<B> {
+	/// Polls for the buffered link actions. Any enqueued action will be propagated to the link
+	/// passed as parameter.
+	///
+	/// This method should behave in a way similar to `Future::poll`. It can register the current
+	/// task and notify later when more actions are ready to be polled. To continue the comparison,
+	/// it is as if this method always returned `Poll::Pending`.
+	pub fn poll_actions(&mut self, cx: &mut Context, link: &mut dyn Link<B>) {
+		loop {
+			let msg = if let Poll::Ready(Some(msg)) = Stream::poll_next(Pin::new(&mut self.rx), cx) {
+				msg
+			} else {
+				break
+			};
+
+			match msg {
+				BlockImportWorkerMsg::BlocksProcessed(imported, count, results) =>
+					link.blocks_processed(imported, count, results),
+				BlockImportWorkerMsg::JustificationImported(who, hash, number, success) =>
+					link.justification_imported(who, &hash, number, success),
+				BlockImportWorkerMsg::RequestJustification(hash, number) =>
+					link.request_justification(&hash, number),
+				BlockImportWorkerMsg::FinalityProofImported(who, block, result) =>
+					link.finality_proof_imported(who, block, result),
+				BlockImportWorkerMsg::RequestFinalityProof(hash, number) =>
+					link.request_finality_proof(&hash, number),
+			}
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use test_client::runtime::Block;
+
+	#[test]
+	fn is_closed() {
+		let (tx, rx) = super::buffered_link::<Block>();
+		assert!(!tx.is_closed());
+		drop(rx);
+		assert!(tx.is_closed());
+	}
+}

substrate/primitives/consensus/common/src/lib.rs

@@ -0,0 +1,135 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate Consensus Common.
+
+// Substrate Demo is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate Consensus Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate Consensus Common.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Common utilities for building and using consensus engines in substrate.
+//!
+//! Much of this crate is _unstable_ and thus the API is likely to undergo
+//! change. Implementors of traits should not rely on the interfaces to remain
+//! the same.
+
+// This provides "unused" building blocks to other crates
+#![allow(dead_code)]
+
+// our error-chain could potentially blow up otherwise
+#![recursion_limit="128"]
+
+#[macro_use] extern crate log;
+
+use std::sync::Arc;
+use std::time::Duration;
+
+use sr_primitives::traits::{Block as BlockT, DigestFor};
+use futures::prelude::*;
+pub use inherents::InherentData;
+
+pub mod block_validation;
+pub mod offline_tracker;
+pub mod error;
+pub mod block_import;
+mod select_chain;
+pub mod import_queue;
+pub mod evaluation;
+
+// block size limit.
+const MAX_BLOCK_SIZE: usize = 4 * 1024 * 1024 + 512;
+
+pub use self::error::Error;
+pub use block_import::{
+	BlockImport, BlockOrigin, ForkChoiceStrategy, ImportedAux, BlockImportParams, BlockCheckParams, ImportResult,
+	JustificationImport, FinalityProofImport,
+};
+pub use select_chain::SelectChain;
+
+/// Block status.
+#[derive(Debug, PartialEq, Eq)]
+pub enum BlockStatus {
+	/// Added to the import queue.
+	Queued,
+	/// Already in the blockchain and the state is available.
+	InChainWithState,
+	/// In the blockchain, but the state is not available.
+	InChainPruned,
+	/// Block or parent is known to be bad.
+	KnownBad,
+	/// Not in the queue or the blockchain.
+	Unknown,
+}
+
+/// Environment producer for a Consensus instance. Creates proposer instance and communication streams.
+pub trait Environment<B: BlockT> {
+	/// The proposer type this creates.
+	type Proposer: Proposer<B>;
+	/// Error which can occur upon creation.
+	type Error: From<Error>;
+
+	/// Initialize the proposal logic on top of a specific header. Provide
+	/// the authorities at that header.
+	fn init(&mut self, parent_header: &B::Header)
+		-> Result<Self::Proposer, Self::Error>;
+}
+
+/// Logic for a proposer.
+///
+/// This will encapsulate creation and evaluation of proposals at a specific
+/// block.
+///
+/// Proposers are generic over bits of "consensus data" which are engine-specific.
+pub trait Proposer<B: BlockT> {
+	/// Error type which can occur when proposing or evaluating.
+	type Error: From<Error> + ::std::fmt::Debug + 'static;
+	/// Future that resolves to a committed proposal.
+	type Create: Future<Output = Result<B, Self::Error>>;
+	/// Create a proposal.
+	fn propose(
+		&mut self,
+		inherent_data: InherentData,
+		inherent_digests: DigestFor<B>,
+		max_duration: Duration,
+	) -> Self::Create;
+}
+
+/// An oracle for when major synchronization work is being undertaken.
+///
+/// Generally, consensus authoring work isn't undertaken while well behind
+/// the head of the chain.
+pub trait SyncOracle {
+	/// Whether the synchronization service is undergoing major sync.
+	/// Returns true if so.
+	fn is_major_syncing(&mut self) -> bool;
+	/// Whether the synchronization service is offline.
+	/// Returns true if so.
+	fn is_offline(&mut self) -> bool;
+}
+
+/// A synchronization oracle for when there is no network.
+#[derive(Clone, Copy, Debug)]
+pub struct NoNetwork;
+
+impl SyncOracle for NoNetwork {
+	fn is_major_syncing(&mut self) -> bool { false }
+	fn is_offline(&mut self) -> bool { false }
+}
+
+impl<T> SyncOracle for Arc<T>
+where T: ?Sized, for<'r> &'r T: SyncOracle
+{
+	fn is_major_syncing(&mut self) -> bool {
+		<&T>::is_major_syncing(&mut &**self)
+	}
+	fn is_offline(&mut self) -> bool {
+		<&T>::is_offline(&mut &**self)
+	}
+}

substrate/primitives/consensus/common/src/offline_tracker.rs

@@ -0,0 +1,135 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Tracks offline validators.
+
+use std::collections::HashMap;
+use std::time::{Instant, Duration};
+
+// time before we report a validator.
+const REPORT_TIME: Duration = Duration::from_secs(60 * 5);
+
+struct Observed {
+	last_round_end: Instant,
+	offline_since: Instant,
+}
+
+impl Observed {
+	fn new() -> Observed {
+		let now = Instant::now();
+		Observed {
+			last_round_end: now,
+			offline_since: now,
+		}
+	}
+
+	fn note_round_end(&mut self, was_online: bool) {
+		let now = Instant::now();
+
+		self.last_round_end = now;
+		if was_online {
+			self.offline_since = now;
+		}
+	}
+
+	fn is_active(&self) -> bool {
+		// can happen if clocks are not monotonic
+		if self.offline_since > self.last_round_end { return true }
+		self.last_round_end.duration_since(self.offline_since) < REPORT_TIME
+	}
+}
+
+/// Tracks offline validators and can issue a report for those offline.
+pub struct OfflineTracker<AuthorityId> {
+	observed: HashMap<AuthorityId, Observed>,
+}
+
+impl<AuthorityId: Eq + Clone + std::hash::Hash> OfflineTracker<AuthorityId> {
+	/// Create a new tracker.
+	pub fn new() -> Self {
+		OfflineTracker { observed: HashMap::new() }
+	}
+
+	/// Note new consensus is starting with the given set of validators.
+	pub fn note_new_block(&mut self, validators: &[AuthorityId]) {
+		use std::collections::HashSet;
+
+		let set: HashSet<_> = validators.iter().cloned().collect();
+		self.observed.retain(|k, _| set.contains(k));
+	}
+
+	/// Note that a round has ended.
+	pub fn note_round_end(&mut self, validator: AuthorityId, was_online: bool) {
+		self.observed.entry(validator)
+			.or_insert_with(Observed::new)
+			.note_round_end(was_online);
+	}
+
+	/// Generate a vector of indices for offline account IDs.
+	pub fn reports(&self, validators: &[AuthorityId]) -> Vec<u32> {
+		validators.iter()
+			.enumerate()
+			.filter_map(|(i, v)| if self.is_online(v) {
+				None
+			} else {
+				Some(i as u32)
+			})
+			.collect()
+	}
+
+	/// Whether reports on a validator set are consistent with our view of things.
+	pub fn check_consistency(&self, validators: &[AuthorityId], reports: &[u32]) -> bool {
+		reports.iter().cloned().all(|r| {
+			let v = match validators.get(r as usize) {
+				Some(v) => v,
+				None => return false,
+			};
+
+			// we must think all validators reported externally are offline.
+			let thinks_online = self.is_online(v);
+			!thinks_online
+		})
+	}
+
+	fn is_online(&self, v: &AuthorityId) -> bool {
+		self.observed.get(v).map(Observed::is_active).unwrap_or(true)
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn validator_offline() {
+		let mut tracker = OfflineTracker::<u64>::new();
+		let v1 = 1;
+		let v2 = 2;
+		let v3 = 3;
+		tracker.note_round_end(v1, true);
+		tracker.note_round_end(v2, true);
+		tracker.note_round_end(v3, true);
+
+		let slash_time = REPORT_TIME + Duration::from_secs(5);
+		tracker.observed.get_mut(&v1).unwrap().offline_since -= slash_time;
+		tracker.observed.get_mut(&v2).unwrap().offline_since -= slash_time;
+
+		assert_eq!(tracker.reports(&[v1, v2, v3]), vec![0, 1]);
+
+		tracker.note_new_block(&[v1, v3]);
+		assert_eq!(tracker.reports(&[v1, v2, v3]), vec![0]);
+	}
+}

substrate/primitives/consensus/common/src/select_chain.rs

@@ -0,0 +1,55 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate Consensus Common.
+
+// Substrate Demo is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate Consensus Common is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate Consensus Common.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::error::Error;
+use sr_primitives::traits::{Block as BlockT, NumberFor};
+
+
+/// The SelectChain trait defines the strategy upon which the head is chosen
+/// if multiple forks are present for an opaque definition of "best" in the
+/// specific chain build.
+///
+/// The Strategy can be customised for the two use cases of authoring new blocks
+/// upon the best chain or which fork to finalise. Unless implemented differently
+/// by default finalisation methods fall back to use authoring, so as a minimum
+/// `_authoring`-functions must be implemented.
+///
+/// Any particular user must make explicit, however, whether they intend to finalise
+/// or author through the using the right function call, as these might differ in
+/// some implementations.
+///
+/// Non-deterministicly finalising chains may only use the `_authoring` functions.
+pub trait SelectChain<Block: BlockT>: Sync + Send + Clone {
+
+	/// Get all leaves of the chain: block hashes that have no children currently.
+	/// Leaves that can never be finalized will not be returned.
+	fn leaves(&self) -> Result<Vec<<Block as BlockT>::Hash>, Error>;
+
+	/// Among those `leaves` deterministically pick one chain as the generally
+	/// best chain to author new blocks upon and probably finalize.
+	fn best_chain(&self) -> Result<<Block as BlockT>::Header, Error>;
+
+	/// Get the best descendent of `target_hash` that we should attempt to
+	/// finalize next, if any. It is valid to return the given `target_hash`
+	/// itself if no better descendent exists.
+	fn finality_target(
+		&self,
+		target_hash: <Block as BlockT>::Hash,
+		_maybe_max_number: Option<NumberFor<Block>>
+	) -> Result<Option<<Block as BlockT>::Hash>, Error> {
+		Ok(Some(target_hash))
+	}
+}

substrate/primitives/consensus/pow/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "substrate-consensus-pow-primitives"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Primitives for Aura consensus"
+edition = "2018"
+
+[dependencies]
+sr-api = { path = "../../sr-api", default-features = false }
+rstd = { package = "sr-std", path = "../../sr-std", default-features = false }
+sr-primitives = { path = "../../sr-primitives", default-features = false }
+primitives = { package = "substrate-primitives", path = "../../core", default-features = false }
+codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] }
+
+[features]
+default = ["std"]
+std = [
+	"rstd/std",
+	"sr-api/std",
+	"sr-primitives/std",
+	"primitives/std",
+	"codec/std",
+]

substrate/primitives/consensus/pow/src/lib.rs

@@ -0,0 +1,64 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Primitives for Substrate Proof-of-Work (PoW) consensus.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+use rstd::vec::Vec;
+use sr_primitives::ConsensusEngineId;
+use codec::Decode;
+
+/// The `ConsensusEngineId` of PoW.
+pub const POW_ENGINE_ID: ConsensusEngineId = [b'p', b'o', b'w', b'_'];
+
+/// Type of seal.
+pub type Seal = Vec<u8>;
+
+/// Define methods that total difficulty should implement.
+pub trait TotalDifficulty {
+	fn increment(&mut self, other: Self);
+}
+
+impl TotalDifficulty for primitives::U256 {
+	fn increment(&mut self, other: Self) {
+		let ret = self.saturating_add(other);
+		*self = ret;
+	}
+}
+
+impl TotalDifficulty for u128 {
+	fn increment(&mut self, other: Self) {
+		let ret = self.saturating_add(other);
+		*self = ret;
+	}
+}
+
+sr_api::decl_runtime_apis! {
+	/// API necessary for timestamp-based difficulty adjustment algorithms.
+	pub trait TimestampApi<Moment: Decode> {
+		/// Return the timestamp in the current block.
+		fn timestamp() -> Moment;
+	}
+
+	/// API for those chains that put their difficulty adjustment algorithm directly
+	/// onto runtime. Note that while putting difficulty adjustment algorithm to
+	/// runtime is safe, putting the PoW algorithm on runtime is not.
+	pub trait DifficultyApi<Difficulty: Decode> {
+		/// Return the target difficulty of the next block.
+		fn difficulty() -> Difficulty;
+	}
+}

substrate/primitives/core/Cargo.toml

@@ -0,0 +1,110 @@
+[package]
+name = "substrate-primitives"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+rstd = { package = "sr-std", path = "../sr-std", default-features = false }
+codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] }
+rustc-hex = { version = "2.0.1", default-features = false }
+log = { version = "0.4.8", default-features = false }
+serde = { version = "1.0.101", optional = true, features = ["derive"] }
+twox-hash = { version = "1.5.0", default-features = false, optional = true }
+byteorder = { version = "1.3.2", default-features = false }
+primitive-types = { version = "0.6", default-features = false, features = ["codec"] }
+impl-serde = { version = "0.2.3", optional = true }
+wasmi = { version = "0.6.2", optional = true }
+hash-db = { version = "0.15.2", default-features = false }
+hash256-std-hasher = { version = "0.15.2", default-features = false }
+ed25519-dalek = { version = "0.9.1",  default-features = false, features = ["u64_backend"], optional = true }
+base58 = { version = "0.1.0", optional = true }
+blake2-rfc = { version = "0.2.18", default-features = false, optional = true }
+schnorrkel = { version = "0.8.5", features = ["preaudit_deprecated"], default-features = false, optional = true }
+rand = { version = "0.7.2", optional = true }
+sha2 = { version = "0.8.0", default-features = false, optional = true }
+substrate-bip39 = { version = "0.3.1", optional = true }
+tiny-bip39 = { version = "0.6.2", optional = true }
+hex = { version = "0.4", default-features = false, optional = true }
+regex = { version = "1.3.1", optional = true }
+num-traits = { version = "0.2.8", default-features = false }
+zeroize = { version = "0.10.1", default-features = false }
+lazy_static = { version = "1.4.0", default-features = false, optional = true }
+parking_lot = { version = "0.9.0", optional = true }
+libsecp256k1 = { version = "0.3.0", default-features = false, optional = true }
+tiny-keccak = { version = "2.0.1", features = ["keccak"], optional = true }
+substrate-debug-derive = { version = "2.0.0", path = "./debug-derive" }
+externalities = { package = "substrate-externalities", path = "../externalities", optional = true }
+primitives-storage = { package = "substrate-primitives-storage", path = "storage", default-features = false }
+runtime-interface = { package = "substrate-runtime-interface", path = "../runtime-interface", default-features = false }
+
+[dev-dependencies]
+substrate-serializer = { path = "../serializer" }
+pretty_assertions = "0.6.1"
+hex-literal = "0.2.1"
+rand = "0.7.2"
+criterion = "0.2.11"
+
+[[bench]]
+name = "bench"
+harness = false
+
+[lib]
+bench = false
+
+[features]
+default = ["std"]
+std = [
+	"full_crypto",
+	"log/std",
+	"wasmi",
+	"lazy_static",
+	"parking_lot",
+	"primitive-types/std",
+	"primitive-types/serde",
+	"primitive-types/byteorder",
+	"primitive-types/rustc-hex",
+	"primitive-types/libc",
+	"impl-serde",
+	"codec/std",
+	"hash256-std-hasher/std",
+	"hash-db/std",
+	"rstd/std",
+	"serde",
+	"rustc-hex/std",
+	"twox-hash/std",
+	"blake2-rfc/std",
+	"ed25519-dalek/std",
+	"hex/std",
+	"base58",
+	"substrate-bip39",
+	"tiny-bip39",
+	"serde",
+	"byteorder/std",
+	"rand",
+	"sha2/std",
+	"schnorrkel/std",
+	"regex",
+	"num-traits/std",
+	"libsecp256k1",
+	"tiny-keccak",
+	"substrate-debug-derive/std",
+	"externalities",
+	"primitives-storage/std",
+	"runtime-interface/std",
+]
+
+# This feature enables all crypto primitives for `no_std` builds like microcontrollers
+# or Intel SGX.
+# For the regular wasm runtime builds this should not be used.
+full_crypto = [
+	"ed25519-dalek",
+	"blake2-rfc",
+	"tiny-keccak",
+	"schnorrkel",
+	"libsecp256k1",
+	"hex",
+	"sha2",
+	"twox-hash",
+	"runtime-interface/disable_target_static_assertions",
+]

substrate/primitives/core/benches/bench.rs

@@ -0,0 +1,95 @@
+// Copyright 2019 Parity Technologies
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+#[macro_use]
+extern crate criterion;
+
+use substrate_primitives as primitives;
+use criterion::{Criterion, black_box, Bencher, Fun};
+use std::time::Duration;
+use primitives::crypto::Pair as _;
+use primitives::hashing::{twox_128, blake2_128};
+
+const MAX_KEY_SIZE: u32 = 32;
+
+fn get_key(key_size: u32) -> Vec<u8> {
+	use rand::SeedableRng;
+	use rand::Rng;
+
+	let rnd: [u8; 32] = rand::rngs::StdRng::seed_from_u64(12).gen();
+	let mut rnd = rnd.iter().cycle();
+
+	(0..key_size)
+		.map(|_| rnd.next().unwrap().clone())
+		.collect()
+}
+
+fn bench_blake2_128(b: &mut Bencher, key: &Vec<u8>) {
+	b.iter(|| {
+		let _a = blake2_128(black_box(key));
+	});
+}
+
+fn bench_twox_128(b: &mut Bencher, key: &Vec<u8>) {
+	b.iter(|| {
+		let _a = twox_128(black_box(key));
+	});
+}
+
+fn bench_hash_128_fix_size(c: &mut Criterion) {
+	let key = get_key(MAX_KEY_SIZE);
+	let blake_fn = Fun::new("blake2_128", bench_blake2_128);
+	let twox_fn = Fun::new("twox_128", bench_twox_128);
+	let fns = vec![blake_fn, twox_fn];
+
+	c.bench_functions("fixed size hashing", fns, key);
+}
+
+fn bench_hash_128_dyn_size(c: &mut Criterion) {
+	let mut keys = Vec::new();
+	for i in (2..MAX_KEY_SIZE).step_by(4) {
+		keys.push(get_key(i).clone())
+	}
+
+	c.bench_function_over_inputs("dyn size hashing - blake2", |b, key| bench_blake2_128(b, &key), keys.clone());
+	c.bench_function_over_inputs("dyn size hashing - twox", |b, key| bench_twox_128(b, &key), keys);
+}
+
+fn bench_ed25519(c: &mut Criterion) {
+	c.bench_function_over_inputs("signing - ed25519", |b, &msg_size| {
+		let msg = (0..msg_size)
+			.map(|_| rand::random::<u8>())
+			.collect::<Vec<_>>();
+		let key = primitives::ed25519::Pair::generate().0;
+		b.iter(|| key.sign(&msg))
+	}, vec![32, 1024, 1024 * 1024]);
+
+	c.bench_function_over_inputs("verifying - ed25519", |b, &msg_size| {
+		let msg = (0..msg_size)
+			.map(|_| rand::random::<u8>())
+			.collect::<Vec<_>>();
+		let key = primitives::ed25519::Pair::generate().0;
+		let sig = key.sign(&msg);
+		let public = key.public();
+		b.iter(|| primitives::ed25519::Pair::verify(&sig, &msg, &public))
+	}, vec![32, 1024, 1024 * 1024]);
+}
+
+criterion_group!{
+    name = benches;
+    config = Criterion::default().warm_up_time(Duration::from_millis(500)).without_plots();
+    targets = bench_hash_128_fix_size, bench_hash_128_dyn_size, bench_ed25519
+}
+criterion_main!(benches);

substrate/primitives/core/debug-derive/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "substrate-debug-derive"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+quote = "1.0.2"
+syn = "1.0.7"
+proc-macro2 = "1.0"
+
+[features]
+std = []
+
+[dev-dependencies]
+

substrate/primitives/core/debug-derive/src/impls.rs

@@ -0,0 +1,217 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use quote::quote;
+use proc_macro2::TokenStream;
+use syn::{Data, DeriveInput, parse_quote};
+
+pub fn debug_derive(ast: DeriveInput) -> proc_macro::TokenStream {
+	let name_str = ast.ident.to_string();
+	let implementation = implementation::derive(&name_str, &ast.data);
+	let name = &ast.ident;
+	let mut generics = ast.generics.clone();
+	let (impl_generics, ty_generics, where_clause) = {
+		let wh = generics.make_where_clause();
+		for t in ast.generics.type_params() {
+			let name = &t.ident;
+			wh.predicates.push(parse_quote!{ #name : core::fmt::Debug });
+		}
+		generics.split_for_impl()
+	};
+	let gen = quote!{
+		impl #impl_generics core::fmt::Debug for #name #ty_generics #where_clause {
+			fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
+				#implementation
+			}
+		}
+	};
+
+	gen.into()
+}
+
+#[cfg(not(feature = "std"))]
+mod implementation {
+	use super::*;
+
+	/// Derive the inner implementation of `Debug::fmt` function.
+	///
+	/// Non-std environment. We do nothing to prevent bloating the size of runtime.
+	/// Implement `Printable` if you need to print the details.
+	pub fn derive(_name_str: &str, _data: &Data) -> TokenStream {
+		quote! {
+			fmt.write_str("<wasm:stripped>")
+		}
+	}
+}
+
+#[cfg(feature = "std")]
+mod implementation {
+	use super::*;
+	use proc_macro2::Span;
+	use syn::{Ident, Index, token::SelfValue};
+
+	/// Derive the inner implementation of `Debug::fmt` function.
+	pub fn derive(name_str: &str, data: &Data) -> TokenStream {
+		match *data {
+			Data::Struct(ref s) => derive_struct(&name_str, &s.fields),
+			Data::Union(ref u) => derive_fields(&name_str, Fields::new(u.fields.named.iter(), None)),
+			Data::Enum(ref e) => derive_enum(&name_str, &e),
+		}
+	}
+
+	enum Fields {
+		Indexed {
+			indices: Vec<Index>,
+		},
+		Unnamed {
+			vars: Vec<Ident>,
+		},
+		Named {
+			names: Vec<Ident>,
+			this: Option<SelfValue>,
+		},
+	}
+
+	impl Fields {
+		fn new<'a>(fields: impl Iterator<Item=&'a syn::Field>, this: Option<SelfValue>) -> Self {
+			let mut indices = vec![];
+			let mut names = vec![];
+
+			for (i, f) in fields.enumerate() {
+				if let Some(ident) = f.ident.clone() {
+					names.push(ident);
+				} else {
+					indices.push(Index::from(i));
+				}
+			}
+
+			if names.is_empty() {
+				Self::Indexed {
+					indices,
+				}
+			} else {
+				Self::Named {
+					names,
+					this,
+				}
+			}
+		}
+	}
+
+	fn derive_fields<'a>(
+		name_str: &str,
+		fields: Fields,
+	) -> TokenStream {
+		match fields {
+			Fields::Named { names, this } => {
+				let names_str: Vec<_> = names.iter()
+					.map(|x| x.to_string())
+					.collect();
+
+				let fields = match this {
+					None => quote! { #( .field(#names_str, #names) )* },
+					Some(this) => quote! { #( .field(#names_str, &#this.#names) )* },
+				};
+
+				quote! {
+					fmt.debug_struct(#name_str)
+						#fields
+						.finish()
+				}
+
+			},
+			Fields::Indexed { indices }  => {
+				quote! {
+					fmt.debug_tuple(#name_str)
+						#( .field(&self.#indices) )*
+						.finish()
+				}
+			},
+			Fields::Unnamed { vars }  => {
+				quote! {
+					fmt.debug_tuple(#name_str)
+						#( .field(#vars) )*
+						.finish()
+				}
+			},
+		}
+	}
+
+	fn derive_enum(
+		name: &str,
+		e: &syn::DataEnum,
+	) -> TokenStream {
+		let v = e.variants
+			.iter()
+			.map(|v| {
+				let name = format!("{}::{}", name, v.ident);
+				let ident = &v.ident;
+				match v.fields {
+					syn::Fields::Named(ref f) => {
+						let names: Vec<_> = f.named.iter().flat_map(|f| f.ident.clone()).collect();
+						let fields_impl = derive_fields(&name, Fields::Named {
+							names: names.clone(),
+							this: None,
+						});
+						(ident, (quote!{ { #( ref #names ),* } }, fields_impl))
+					},
+					syn::Fields::Unnamed(ref f) => {
+						let names = f.unnamed.iter()
+							.enumerate()
+							.map(|(id, _)| Ident::new(&format!("a{}", id), Span::call_site()))
+							.collect::<Vec<_>>();
+						let fields_impl = derive_fields(&name, Fields::Unnamed { vars: names.clone() });
+						(ident, (quote! { ( #( ref #names ),* ) }, fields_impl))
+					},
+					syn::Fields::Unit => {
+						let fields_impl = derive_fields(&name, Fields::Indexed { indices: vec![] });
+						(ident, (quote! { }, fields_impl))
+					},
+				}
+			});
+
+		type Vecs<A, B> = (Vec<A>, Vec<B>);
+		let (variants, others): Vecs<_, _> = v.unzip();
+		let (match_fields, variants_impl): Vecs<_, _> = others.into_iter().unzip();
+
+		quote! {
+			match self {
+				#( Self::#variants #match_fields => #variants_impl, )*
+				_ => Ok(()),
+			}
+		}
+	}
+
+	fn derive_struct(
+		name_str: &str,
+		fields: &syn::Fields,
+	) -> TokenStream {
+		match *fields {
+			syn::Fields::Named(ref f) => derive_fields(
+				name_str,
+				Fields::new(f.named.iter(), Some(syn::Token!(self)(Span::call_site()))),
+			),
+			syn::Fields::Unnamed(ref f) => derive_fields(
+				name_str,
+				Fields::new(f.unnamed.iter(), None),
+			),
+			syn::Fields::Unit => derive_fields(
+				name_str,
+				Fields::Indexed { indices: vec![] },
+			),
+		}
+	}
+}

substrate/primitives/core/debug-derive/src/lib.rs

@@ -0,0 +1,44 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Macros to derive runtime debug implementation.
+//!
+//! This custom derive implements a `core::fmt::Debug` trait,
+//! but in case the `std` feature is enabled the implementation
+//! will actually print out the structure as regular `derive(Debug)`
+//! would do. If `std` is disabled the implementation will be empty.
+//!
+//! This behaviour is useful to prevent bloating the runtime WASM
+//! blob from unneeded code.
+//!
+//! ```rust
+//! #[derive(substrate_debug_derive::RuntimeDebug)]
+//!	struct MyStruct;
+//!
+//!	assert_eq!(format!("{:?}", MyStruct), "MyStruct");
+//! ```
+
+extern crate proc_macro;
+
+mod impls;
+
+use proc_macro::TokenStream;
+
+#[proc_macro_derive(RuntimeDebug)]
+pub fn debug_derive(input: TokenStream) -> TokenStream {
+   impls::debug_derive(syn::parse_macro_input!(input))
+}
+

substrate/primitives/core/debug-derive/tests/tests.rs

@@ -0,0 +1,63 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use substrate_debug_derive::RuntimeDebug;
+
+#[derive(RuntimeDebug)]
+struct Unnamed(u64, String);
+
+#[derive(RuntimeDebug)]
+struct Named {
+	a: u64,
+	b: String,
+}
+
+#[derive(RuntimeDebug)]
+enum EnumLongName<A> {
+	A,
+	B(A, String),
+	VariantLongName {
+		a: A,
+		b: String,
+	},
+}
+
+
+#[test]
+fn should_display_proper_debug() {
+	use self::EnumLongName as Enum;
+
+	assert_eq!(
+		format!("{:?}", Unnamed(1, "abc".into())),
+		"Unnamed(1, \"abc\")"
+	);
+	assert_eq!(
+		format!("{:?}", Named { a: 1, b: "abc".into() }),
+		"Named { a: 1, b: \"abc\" }"
+	);
+	assert_eq!(
+		format!("{:?}", Enum::<u64>::A),
+		"EnumLongName::A"
+	);
+	assert_eq!(
+		format!("{:?}", Enum::B(1, "abc".into())),
+		"EnumLongName::B(1, \"abc\")"
+	);
+	assert_eq!(
+		format!("{:?}", Enum::VariantLongName { a: 1, b: "abc".into() }),
+		"EnumLongName::VariantLongName { a: 1, b: \"abc\" }"
+	);
+}

substrate/primitives/core/src/changes_trie.rs

@@ -0,0 +1,294 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Substrate changes trie configuration.
+
+#[cfg(any(feature = "std", test))]
+use serde::{Serialize, Deserialize};
+use codec::{Encode, Decode};
+use num_traits::Zero;
+
+/// Substrate changes trie configuration.
+#[cfg_attr(any(feature = "std", test), derive(Serialize, Deserialize))]
+#[derive(Debug, Clone, PartialEq, Eq, Default, Encode, Decode)]
+pub struct ChangesTrieConfiguration {
+	/// Interval (in blocks) at which level1-digests are created. Digests are not
+	/// created when this is less or equal to 1.
+	pub digest_interval: u32,
+	/// Maximal number of digest levels in hierarchy. 0 means that digests are not
+	/// created at all (even level1 digests). 1 means only level1-digests are created.
+	/// 2 means that every digest_interval^2 there will be a level2-digest, and so on.
+	/// Please ensure that maximum digest interval (i.e. digest_interval^digest_levels)
+	/// is within `u32` limits. Otherwise you'll never see digests covering such intervals
+	/// && maximal digests interval will be truncated to the last interval that fits
+	/// `u32` limits.
+	pub digest_levels: u32,
+}
+
+impl ChangesTrieConfiguration {
+	/// Create new configuration given digest interval and levels.
+	pub fn new(digest_interval: u32, digest_levels: u32) -> Self {
+		Self { digest_interval, digest_levels }
+	}
+
+	/// Is digest build enabled?
+	pub fn is_digest_build_enabled(&self) -> bool {
+		self.digest_interval > 1 && self.digest_levels > 0
+	}
+
+	/// Do we need to build digest at given block?
+	pub fn is_digest_build_required_at_block<Number>(
+		&self,
+		zero: Number,
+		block: Number,
+	) -> bool
+		where
+			Number: From<u32> + PartialEq +
+			::rstd::ops::Rem<Output=Number> + ::rstd::ops::Sub<Output=Number> +
+			::rstd::cmp::PartialOrd + Zero,
+	{
+		block > zero
+			&& self.is_digest_build_enabled()
+			&& ((block - zero) % self.digest_interval.into()).is_zero()
+	}
+
+	/// Returns max digest interval. One if digests are not created at all.
+	pub fn max_digest_interval(&self) -> u32 {
+		if !self.is_digest_build_enabled() {
+			return 1;
+		}
+
+		// we'll get >1 loop iteration only when bad configuration parameters are selected
+		let mut current_level = self.digest_levels;
+		loop {
+			if let Some(max_digest_interval) = self.digest_interval.checked_pow(current_level) {
+				return max_digest_interval;
+			}
+
+			current_level = current_level - 1;
+		}
+	}
+
+	/// Returns max level digest block number that has been created at block <= passed block number.
+	///
+	/// Returns None if digests are not created at all.
+	pub fn prev_max_level_digest_block<Number>(
+		&self,
+		zero: Number,
+		block: Number,
+	) -> Option<Number>
+		where
+			Number: Clone + From<u32> + PartialOrd + PartialEq +
+			::rstd::ops::Add<Output=Number> + ::rstd::ops::Sub<Output=Number> +
+			::rstd::ops::Div<Output=Number> + ::rstd::ops::Mul<Output=Number> + Zero,
+	{
+		if block <= zero {
+			return None;
+		}
+
+		let (next_begin, next_end) = self.next_max_level_digest_range(zero.clone(), block.clone())?;
+
+		// if 'next' digest includes our block, then it is a also a previous digest
+		if next_end == block {
+			return Some(block);
+		}
+
+		// if previous digest ends at zero block, then there are no previous digest
+		let prev_end = next_begin - 1.into();
+		if prev_end == zero {
+			None
+		} else {
+			Some(prev_end)
+		}
+	}
+
+	/// Returns max level digest blocks range (inclusive) which includes passed block.
+	///
+	/// Returns None if digests are not created at all.
+	/// It will return the first max-level digest if block is <= zero.
+	pub fn next_max_level_digest_range<Number>(
+		&self,
+		zero: Number,
+		mut block: Number,
+	) -> Option<(Number, Number)>
+		where
+			Number: Clone + From<u32> + PartialOrd + PartialEq +
+			::rstd::ops::Add<Output=Number> + ::rstd::ops::Sub<Output=Number> +
+			::rstd::ops::Div<Output=Number> + ::rstd::ops::Mul<Output=Number>,
+	{
+		if !self.is_digest_build_enabled() {
+			return None;
+		}
+
+		if block <= zero {
+			block = zero.clone() + 1.into();
+		}
+
+		let max_digest_interval: Number = self.max_digest_interval().into();
+		let max_digests_since_zero = (block.clone() - zero.clone()) / max_digest_interval.clone();
+		if max_digests_since_zero == 0.into() {
+			return Some((zero.clone() + 1.into(), zero + max_digest_interval));
+		}
+		let last_max_digest_block = zero + max_digests_since_zero * max_digest_interval.clone();
+		Some(if block == last_max_digest_block {
+			(block.clone() - max_digest_interval + 1.into(), block)
+		} else {
+			(last_max_digest_block.clone() + 1.into(), last_max_digest_block + max_digest_interval)
+		})
+	}
+
+	/// Returns Some if digest must be built at given block number.
+	/// The tuple is:
+	/// (
+	///  digest level
+	///  digest interval (in blocks)
+	///  step between blocks we're interested in when digest is built
+	/// )
+	pub fn digest_level_at_block<Number>(&self, zero: Number, block: Number) -> Option<(u32, u32, u32)>
+		where
+			Number: Clone + From<u32> + PartialEq +
+			::rstd::ops::Rem<Output=Number> + ::rstd::ops::Sub<Output=Number> +
+			::rstd::cmp::PartialOrd + Zero,
+	{
+		if !self.is_digest_build_required_at_block(zero.clone(), block.clone()) {
+			return None;
+		}
+
+		let relative_block = block - zero;
+		let mut digest_interval = self.digest_interval;
+		let mut current_level = 1u32;
+		let mut digest_step = 1u32;
+		while current_level < self.digest_levels {
+			let new_digest_interval = match digest_interval.checked_mul(self.digest_interval) {
+				Some(new_digest_interval) if (relative_block.clone() % new_digest_interval.into()).is_zero()
+					=> new_digest_interval,
+				_ => break,
+			};
+
+			digest_step = digest_interval;
+			digest_interval = new_digest_interval;
+			current_level = current_level + 1;
+		}
+
+		Some((
+			current_level,
+			digest_interval,
+			digest_step,
+		))
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::ChangesTrieConfiguration;
+
+	fn config(interval: u32, levels: u32) -> ChangesTrieConfiguration {
+		ChangesTrieConfiguration {
+			digest_interval: interval,
+			digest_levels: levels,
+		}
+	}
+
+	#[test]
+	fn is_digest_build_enabled_works() {
+		assert!(!config(0, 100).is_digest_build_enabled());
+		assert!(!config(1, 100).is_digest_build_enabled());
+		assert!(config(2, 100).is_digest_build_enabled());
+		assert!(!config(100, 0).is_digest_build_enabled());
+		assert!(config(100, 1).is_digest_build_enabled());
+	}
+
+	#[test]
+	fn is_digest_build_required_at_block_works() {
+		fn test_with_zero(zero: u64) {
+			assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 0u64));
+			assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 1u64));
+			assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 2u64));
+			assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 4u64));
+			assert!(config(8, 4).is_digest_build_required_at_block(zero, zero + 8u64));
+			assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 9u64));
+			assert!(config(8, 4).is_digest_build_required_at_block(zero, zero + 64u64));
+			assert!(config(8, 4).is_digest_build_required_at_block(zero, zero + 64u64));
+			assert!(config(8, 4).is_digest_build_required_at_block(zero, zero + 512u64));
+			assert!(config(8, 4).is_digest_build_required_at_block(zero, zero + 4096u64));
+			assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 4103u64));
+			assert!(config(8, 4).is_digest_build_required_at_block(zero, zero + 4104u64));
+			assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 4108u64));
+		}
+
+		test_with_zero(0);
+		test_with_zero(8);
+		test_with_zero(17);
+	}
+
+	#[test]
+	fn digest_level_at_block_works() {
+		fn test_with_zero(zero: u64) {
+			assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 0u64), None);
+			assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 7u64), None);
+			assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 63u64), None);
+			assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 8u64), Some((1, 8, 1)));
+			assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 64u64), Some((2, 64, 8)));
+			assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 512u64), Some((3, 512, 64)));
+			assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 4096u64), Some((4, 4096, 512)));
+			assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 4112u64), Some((1, 8, 1)));
+		}
+
+		test_with_zero(0);
+		test_with_zero(8);
+		test_with_zero(17);
+	}
+
+	#[test]
+	fn max_digest_interval_works() {
+		assert_eq!(config(0, 0).max_digest_interval(), 1);
+		assert_eq!(config(2, 2).max_digest_interval(), 4);
+		assert_eq!(config(8, 4).max_digest_interval(), 4096);
+		assert_eq!(config(::std::u32::MAX, 1024).max_digest_interval(), ::std::u32::MAX);
+	}
+
+	#[test]
+	fn next_max_level_digest_range_works() {
+		assert_eq!(config(0, 0).next_max_level_digest_range(0u64, 16), None);
+		assert_eq!(config(1, 1).next_max_level_digest_range(0u64, 16), None);
+		assert_eq!(config(2, 1).next_max_level_digest_range(0u64, 16), Some((15, 16)));
+		assert_eq!(config(4, 1).next_max_level_digest_range(0u64, 16), Some((13, 16)));
+		assert_eq!(config(32, 1).next_max_level_digest_range(0u64, 16), Some((1, 32)));
+		assert_eq!(config(2, 3).next_max_level_digest_range(0u64, 10), Some((9, 16)));
+		assert_eq!(config(2, 3).next_max_level_digest_range(0u64, 8), Some((1, 8)));
+		assert_eq!(config(2, 1).next_max_level_digest_range(1u64, 1), Some((2, 3)));
+		assert_eq!(config(2, 2).next_max_level_digest_range(7u64, 9), Some((8, 11)));
+
+		assert_eq!(config(2, 2).next_max_level_digest_range(7u64, 5), Some((8, 11)));
+	}
+
+	#[test]
+	fn prev_max_level_digest_block_works() {
+		assert_eq!(config(0, 0).prev_max_level_digest_block(0u64, 16), None);
+		assert_eq!(config(1, 1).prev_max_level_digest_block(0u64, 16), None);
+		assert_eq!(config(2, 1).prev_max_level_digest_block(0u64, 16), Some(16));
+		assert_eq!(config(4, 1).prev_max_level_digest_block(0u64, 16), Some(16));
+		assert_eq!(config(4, 2).prev_max_level_digest_block(0u64, 16), Some(16));
+		assert_eq!(config(4, 2).prev_max_level_digest_block(0u64, 17), Some(16));
+		assert_eq!(config(4, 2).prev_max_level_digest_block(0u64, 33), Some(32));
+		assert_eq!(config(32, 1).prev_max_level_digest_block(0u64, 16), None);
+		assert_eq!(config(2, 3).prev_max_level_digest_block(0u64, 10), Some(8));
+		assert_eq!(config(2, 3).prev_max_level_digest_block(0u64, 8), Some(8));
+		assert_eq!(config(2, 2).prev_max_level_digest_block(7u64, 8), None);
+
+		assert_eq!(config(2, 2).prev_max_level_digest_block(7u64, 5), None);
+	}
+}

substrate/primitives/core/src/ecdsa.rs

@@ -0,0 +1,616 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+// tag::description[]
+//! Simple ECDSA API.
+// end::description[]
+
+#[cfg(feature = "full_crypto")]
+use rstd::vec::Vec;
+
+use rstd::cmp::Ordering;
+use codec::{Encode, Decode};
+
+#[cfg(feature = "full_crypto")]
+use core::convert::{TryFrom, TryInto};
+#[cfg(feature = "std")]
+use substrate_bip39::seed_from_entropy;
+#[cfg(feature = "std")]
+use bip39::{Mnemonic, Language, MnemonicType};
+#[cfg(feature = "full_crypto")]
+use crate::{hashing::blake2_256, crypto::{Pair as TraitPair, DeriveJunction, SecretStringError}};
+#[cfg(feature = "std")]
+use crate::crypto::Ss58Codec;
+#[cfg(feature = "std")]
+use serde::{de, Serializer, Serialize, Deserializer, Deserialize};
+use crate::crypto::{Public as TraitPublic, UncheckedFrom, CryptoType, Derive};
+#[cfg(feature = "full_crypto")]
+use secp256k1::{PublicKey, SecretKey};
+
+/// A secret seed (which is bytewise essentially equivalent to a SecretKey).
+///
+/// We need it as a different type because `Seed` is expected to be AsRef<[u8]>.
+#[cfg(feature = "full_crypto")]
+type Seed = [u8; 32];
+
+/// The ECDSA 33-byte compressed public key.
+#[derive(Clone, Encode, Decode)]
+pub struct Public(pub [u8; 33]);
+
+impl PartialOrd for Public {
+	fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+		Some(self.cmp(other))
+	}
+}
+
+impl Ord for Public {
+	fn cmp(&self, other: &Self) -> Ordering {
+		self.0[..].cmp(&other.0[..])
+	}
+}
+
+impl PartialEq for Public {
+	fn eq(&self, other: &Self) -> bool {
+		&self.0[..] == &other.0[..]
+	}
+}
+
+impl Eq for Public {}
+
+impl Default for Public {
+	fn default() -> Self {
+		Public([0u8; 33])
+	}
+}
+
+/// A key pair.
+#[cfg(feature = "full_crypto")]
+#[derive(Clone)]
+pub struct Pair {
+	public: PublicKey,
+	secret: SecretKey,
+}
+
+impl AsRef<[u8; 33]> for Public {
+	fn as_ref(&self) -> &[u8; 33] {
+		&self.0
+	}
+}
+
+impl AsRef<[u8]> for Public {
+	fn as_ref(&self) -> &[u8] {
+		&self.0[..]
+	}
+}
+
+impl AsMut<[u8]> for Public {
+	fn as_mut(&mut self) -> &mut [u8] {
+		&mut self.0[..]
+	}
+}
+
+impl rstd::convert::TryFrom<&[u8]> for Public {
+	type Error = ();
+
+	fn try_from(data: &[u8]) -> Result<Self, Self::Error> {
+		if data.len() == 33 {
+			let mut inner = [0u8; 33];
+			inner.copy_from_slice(data);
+			Ok(Public(inner))
+		} else {
+			Err(())
+		}
+	}
+}
+
+impl From<Public> for [u8; 33] {
+	fn from(x: Public) -> Self {
+		x.0
+	}
+}
+
+#[cfg(feature = "full_crypto")]
+impl From<Pair> for Public {
+	fn from(x: Pair) -> Self {
+		x.public()
+	}
+}
+
+impl UncheckedFrom<[u8; 33]> for Public {
+	fn unchecked_from(x: [u8; 33]) -> Self {
+		Public(x)
+	}
+}
+
+#[cfg(feature = "std")]
+impl std::fmt::Display for Public {
+	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+		write!(f, "{}", self.to_ss58check())
+	}
+}
+
+#[cfg(feature = "std")]
+impl std::fmt::Debug for Public {
+	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+		let s = self.to_ss58check();
+		write!(f, "{} ({}...)", crate::hexdisplay::HexDisplay::from(&&self.0[..]), &s[0..8])
+	}
+}
+
+#[cfg(feature = "std")]
+impl Serialize for Public {
+	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> where S: Serializer {
+		serializer.serialize_str(&self.to_ss58check())
+	}
+}
+
+#[cfg(feature = "std")]
+impl<'de> Deserialize<'de> for Public {
+	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> where D: Deserializer<'de> {
+		Public::from_ss58check(&String::deserialize(deserializer)?)
+			.map_err(|e| de::Error::custom(format!("{:?}", e)))
+	}
+}
+
+#[cfg(feature = "full_crypto")]
+impl rstd::hash::Hash for Public {
+	fn hash<H: rstd::hash::Hasher>(&self, state: &mut H) {
+		self.0.hash(state);
+	}
+}
+
+/// A signature (a 512-bit value).
+#[derive(Encode, Decode)]
+pub struct Signature([u8; 65]);
+
+impl rstd::convert::TryFrom<&[u8]> for Signature {
+	type Error = ();
+
+	fn try_from(data: &[u8]) -> Result<Self, Self::Error> {
+		if data.len() == 65 {
+			let mut inner = [0u8; 65];
+			inner.copy_from_slice(data);
+			Ok(Signature(inner))
+		} else {
+			Err(())
+		}
+	}
+}
+
+impl Clone for Signature {
+	fn clone(&self) -> Self {
+		let mut r = [0u8; 65];
+		r.copy_from_slice(&self.0[..]);
+		Signature(r)
+	}
+}
+
+impl Default for Signature {
+	fn default() -> Self {
+		Signature([0u8; 65])
+	}
+}
+
+impl PartialEq for Signature {
+	fn eq(&self, b: &Self) -> bool {
+		self.0[..] == b.0[..]
+	}
+}
+
+impl Eq for Signature {}
+
+impl From<Signature> for [u8; 65] {
+	fn from(v: Signature) -> [u8; 65] {
+		v.0
+	}
+}
+
+impl AsRef<[u8; 65]> for Signature {
+	fn as_ref(&self) -> &[u8; 65] {
+		&self.0
+	}
+}
+
+impl AsRef<[u8]> for Signature {
+	fn as_ref(&self) -> &[u8] {
+		&self.0[..]
+	}
+}
+
+impl AsMut<[u8]> for Signature {
+	fn as_mut(&mut self) -> &mut [u8] {
+		&mut self.0[..]
+	}
+}
+
+#[cfg(feature = "std")]
+impl std::fmt::Debug for Signature {
+	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+		write!(f, "{}", crate::hexdisplay::HexDisplay::from(&self.0))
+	}
+}
+
+#[cfg(feature = "full_crypto")]
+impl rstd::hash::Hash for Signature {
+	fn hash<H: rstd::hash::Hasher>(&self, state: &mut H) {
+		rstd::hash::Hash::hash(&self.0[..], state);
+	}
+}
+
+impl Signature {
+	/// A new instance from the given 65-byte `data`.
+	///
+	/// NOTE: No checking goes on to ensure this is a real signature. Only use it if
+	/// you are certain that the array actually is a signature. GIGO!
+	pub fn from_raw(data: [u8; 65]) -> Signature {
+		Signature(data)
+	}
+
+	/// Recover the public key from this signature and a message.
+	#[cfg(feature = "full_crypto")]
+	pub fn recover<M: AsRef<[u8]>>(&self, message: M) -> Option<Public> {
+		let message = secp256k1::Message::parse(&blake2_256(message.as_ref()));
+		let sig: (_, _) = self.try_into().ok()?;
+		secp256k1::recover(&message, &sig.0, &sig.1).ok()
+			.map(|recovered| Public(recovered.serialize_compressed()))
+	}
+}
+
+#[cfg(feature = "full_crypto")]
+impl From<(secp256k1::Signature, secp256k1::RecoveryId)> for Signature {
+	fn from(x: (secp256k1::Signature, secp256k1::RecoveryId)) -> Signature {
+		let mut r = Self::default();
+		r.0[0..64].copy_from_slice(&x.0.serialize()[..]);
+		r.0[64] = x.1.serialize();
+		r
+	}
+}
+
+#[cfg(feature = "full_crypto")]
+impl<'a> TryFrom<&'a Signature> for (secp256k1::Signature, secp256k1::RecoveryId) {
+	type Error = ();
+	fn try_from(x: &'a Signature) -> Result<(secp256k1::Signature, secp256k1::RecoveryId), Self::Error> {
+		Ok((
+			secp256k1::Signature::parse_slice(&x.0[0..64]).expect("hardcoded to 64 bytes; qed"),
+			secp256k1::RecoveryId::parse(x.0[64]).map_err(|_| ())?,
+		))
+	}
+}
+
+/// An error type for SS58 decoding.
+#[cfg(feature = "std")]
+#[derive(Clone, Copy, Eq, PartialEq, Debug)]
+pub enum PublicError {
+	/// Bad alphabet.
+	BadBase58,
+	/// Bad length.
+	BadLength,
+	/// Unknown version.
+	UnknownVersion,
+	/// Invalid checksum.
+	InvalidChecksum,
+}
+
+impl Public {
+	/// A new instance from the given 33-byte `data`.
+	///
+	/// NOTE: No checking goes on to ensure this is a real public key. Only use it if
+	/// you are certain that the array actually is a pubkey. GIGO!
+	pub fn from_raw(data: [u8; 33]) -> Self {
+		Public(data)
+	}
+
+	/// Return a slice filled with raw data.
+	pub fn as_array_ref(&self) -> &[u8; 33] {
+		self.as_ref()
+	}
+}
+
+impl TraitPublic for Public {
+	/// A new instance from the given slice that should be 33 bytes long.
+	///
+	/// NOTE: No checking goes on to ensure this is a real public key. Only use it if
+	/// you are certain that the array actually is a pubkey. GIGO!
+	fn from_slice(data: &[u8]) -> Self {
+		let mut r = [0u8; 33];
+		r.copy_from_slice(data);
+		Public(r)
+	}
+}
+
+impl Derive for Public {}
+
+/// Derive a single hard junction.
+#[cfg(feature = "full_crypto")]
+fn derive_hard_junction(secret_seed: &Seed, cc: &[u8; 32]) -> Seed {
+	("Secp256k1HDKD", secret_seed, cc).using_encoded(|data| {
+		let mut res = [0u8; 32];
+		res.copy_from_slice(blake2_rfc::blake2b::blake2b(32, &[], data).as_bytes());
+		res
+	})
+}
+
+/// An error when deriving a key.
+#[cfg(feature = "full_crypto")]
+pub enum DeriveError {
+	/// A soft key was found in the path (and is unsupported).
+	SoftKeyInPath,
+}
+
+#[cfg(feature = "full_crypto")]
+impl TraitPair for Pair {
+	type Public = Public;
+	type Seed = Seed;
+	type Signature = Signature;
+	type DeriveError = DeriveError;
+
+	/// Generate new secure (random) key pair and provide the recovery phrase.
+	///
+	/// You can recover the same key later with `from_phrase`.
+	#[cfg(feature = "std")]
+	fn generate_with_phrase(password: Option<&str>) -> (Pair, String, Seed) {
+		let mnemonic = Mnemonic::new(MnemonicType::Words12, Language::English);
+		let phrase = mnemonic.phrase();
+		let (pair, seed) = Self::from_phrase(phrase, password)
+			.expect("All phrases generated by Mnemonic are valid; qed");
+		(
+			pair,
+			phrase.to_owned(),
+			seed,
+		)
+	}
+
+	/// Generate key pair from given recovery phrase and password.
+	#[cfg(feature = "std")]
+	fn from_phrase(phrase: &str, password: Option<&str>) -> Result<(Pair, Seed), SecretStringError> {
+		let big_seed = seed_from_entropy(
+			Mnemonic::from_phrase(phrase, Language::English)
+				.map_err(|_| SecretStringError::InvalidPhrase)?.entropy(),
+			password.unwrap_or(""),
+		).map_err(|_| SecretStringError::InvalidSeed)?;
+		let mut seed = Seed::default();
+		seed.copy_from_slice(&big_seed[0..32]);
+		Self::from_seed_slice(&big_seed[0..32]).map(|x| (x, seed))
+	}
+
+	/// Make a new key pair from secret seed material.
+	///
+	/// You should never need to use this; generate(), generate_with_phrase
+	fn from_seed(seed: &Seed) -> Pair {
+		Self::from_seed_slice(&seed[..]).expect("seed has valid length; qed")
+	}
+
+	/// Make a new key pair from secret seed material. The slice must be 32 bytes long or it
+	/// will return `None`.
+	///
+	/// You should never need to use this; generate(), generate_with_phrase
+	fn from_seed_slice(seed_slice: &[u8]) -> Result<Pair, SecretStringError> {
+		let secret = SecretKey::parse_slice(seed_slice)
+			.map_err(|_| SecretStringError::InvalidSeedLength)?;
+		let public = PublicKey::from_secret_key(&secret);
+		Ok(Pair{ secret, public })
+	}
+
+	/// Derive a child key from a series of given junctions.
+	fn derive<Iter: Iterator<Item=DeriveJunction>>(&self,
+		path: Iter,
+		_seed: Option<Seed>
+	) -> Result<(Pair, Option<Seed>), DeriveError> {
+		let mut acc = self.secret.serialize();
+		for j in path {
+			match j {
+				DeriveJunction::Soft(_cc) => return Err(DeriveError::SoftKeyInPath),
+				DeriveJunction::Hard(cc) => acc = derive_hard_junction(&acc, &cc),
+			}
+		}
+		Ok((Self::from_seed(&acc), Some(acc)))
+	}
+
+	/// Get the public key.
+	fn public(&self) -> Public {
+		Public(self.public.serialize_compressed())
+	}
+
+	/// Sign a message.
+	fn sign(&self, message: &[u8]) -> Signature {
+		let message = secp256k1::Message::parse(&blake2_256(message));
+		secp256k1::sign(&message, &self.secret).into()
+	}
+
+	/// Verify a signature on a message. Returns true if the signature is good.
+	fn verify<M: AsRef<[u8]>>(sig: &Self::Signature, message: M, pubkey: &Self::Public) -> bool {
+		let message = secp256k1::Message::parse(&blake2_256(message.as_ref()));
+		let sig: (_, _) = match sig.try_into() { Ok(x) => x, _ => return false };
+		match secp256k1::recover(&message, &sig.0, &sig.1) {
+			Ok(actual) => &pubkey.0[..] == &actual.serialize_compressed()[..],
+			_ => false,
+		}
+	}
+
+	/// Verify a signature on a message. Returns true if the signature is good.
+	///
+	/// This doesn't use the type system to ensure that `sig` and `pubkey` are the correct
+	/// size. Use it only if you're coming from byte buffers and need the speed.
+	fn verify_weak<P: AsRef<[u8]>, M: AsRef<[u8]>>(sig: &[u8], message: M, pubkey: P) -> bool {
+		let message = secp256k1::Message::parse(&blake2_256(message.as_ref()));
+		if sig.len() != 65 { return false }
+		let ri = match secp256k1::RecoveryId::parse(sig[64]) { Ok(x) => x, _ => return false };
+		let sig = match secp256k1::Signature::parse_slice(&sig[0..64]) { Ok(x) => x, _ => return false };
+		match secp256k1::recover(&message, &sig, &ri) {
+			Ok(actual) => pubkey.as_ref() == &actual.serialize_compressed()[..],
+			_ => false,
+		}
+	}
+
+	/// Return a vec filled with raw data.
+	fn to_raw_vec(&self) -> Vec<u8> {
+		self.seed().to_vec()
+	}
+}
+
+#[cfg(feature = "full_crypto")]
+impl Pair {
+	/// Get the seed for this key.
+	pub fn seed(&self) -> Seed {
+		self.secret.serialize()
+	}
+
+	/// Exactly as `from_string` except that if no matches are found then, the the first 32
+	/// characters are taken (padded with spaces as necessary) and used as the MiniSecretKey.
+	#[cfg(feature = "std")]
+	pub fn from_legacy_string(s: &str, password_override: Option<&str>) -> Pair {
+		Self::from_string(s, password_override).unwrap_or_else(|_| {
+			let mut padded_seed: Seed = [' ' as u8; 32];
+			let len = s.len().min(32);
+			padded_seed[..len].copy_from_slice(&s.as_bytes()[..len]);
+			Self::from_seed(&padded_seed)
+		})
+	}
+}
+
+impl CryptoType for Public {
+	#[cfg(feature="full_crypto")]
+	type Pair = Pair;
+}
+
+impl CryptoType for Signature {
+	#[cfg(feature="full_crypto")]
+	type Pair = Pair;
+}
+
+#[cfg(feature="full_crypto")]
+impl CryptoType for Pair {
+	type Pair = Pair;
+}
+
+#[cfg(test)]
+mod test {
+	use super::*;
+	use hex_literal::hex;
+	use crate::crypto::DEV_PHRASE;
+
+	#[test]
+	fn default_phrase_should_be_used() {
+		assert_eq!(
+			Pair::from_string("//Alice///password", None).unwrap().public(),
+			Pair::from_string(&format!("{}//Alice", DEV_PHRASE), Some("password")).unwrap().public(),
+		);
+	}
+
+	#[test]
+	fn seed_and_derive_should_work() {
+		let seed = hex!("9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60");
+		let pair = Pair::from_seed(&seed);
+		assert_eq!(pair.seed(), seed);
+		let path = vec![DeriveJunction::Hard([0u8; 32])];
+		let derived = pair.derive(path.into_iter(), None).ok().unwrap();
+		assert_eq!(
+			derived.0.seed(),
+			hex!("b8eefc4937200a8382d00050e050ced2d4ab72cc2ef1b061477afb51564fdd61")
+		);
+	}
+
+	#[test]
+	fn test_vector_should_work() {
+		let pair = Pair::from_seed(
+			&hex!("9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60")
+		);
+		let public = pair.public();
+		assert_eq!(public, Public::from_raw(
+			hex!("028db55b05db86c0b1786ca49f095d76344c9e6056b2f02701a7e7f3c20aabfd91")
+		));
+		let message = b"";
+		let signature = hex!("3dde91174bd9359027be59a428b8146513df80a2a3c7eda2194f64de04a69ab97b753169e94db6ffd50921a2668a48b94ca11e3d32c1ff19cfe88890aa7e8f3c00");
+		let signature = Signature::from_raw(signature);
+		assert!(&pair.sign(&message[..]) == &signature);
+		assert!(Pair::verify(&signature, &message[..], &public));
+	}
+
+	#[test]
+	fn test_vector_by_string_should_work() {
+		let pair = Pair::from_string(
+			"0x9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60",
+			None
+		).unwrap();
+		let public = pair.public();
+		assert_eq!(public, Public::from_raw(
+			hex!("028db55b05db86c0b1786ca49f095d76344c9e6056b2f02701a7e7f3c20aabfd91")
+		));
+		let message = b"";
+		let signature = hex!("3dde91174bd9359027be59a428b8146513df80a2a3c7eda2194f64de04a69ab97b753169e94db6ffd50921a2668a48b94ca11e3d32c1ff19cfe88890aa7e8f3c00");
+		let signature = Signature::from_raw(signature);
+		assert!(&pair.sign(&message[..]) == &signature);
+		assert!(Pair::verify(&signature, &message[..], &public));
+	}
+
+	#[test]
+	fn generated_pair_should_work() {
+		let (pair, _) = Pair::generate();
+		let public = pair.public();
+		let message = b"Something important";
+		let signature = pair.sign(&message[..]);
+		assert!(Pair::verify(&signature, &message[..], &public));
+		assert!(!Pair::verify(&signature, b"Something else", &public));
+	}
+
+	#[test]
+	fn seeded_pair_should_work() {
+		let pair = Pair::from_seed(b"12345678901234567890123456789012");
+		let public = pair.public();
+		assert_eq!(public, Public::from_raw(
+			hex!("035676109c54b9a16d271abeb4954316a40a32bcce023ac14c8e26e958aa68fba9")
+		));
+		let message = hex!("2f8c6129d816cf51c374bc7f08c3e63ed156cf78aefb4a6550d97b87997977ee00000000000000000200d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a4500000000000000");
+		let signature = pair.sign(&message[..]);
+		println!("Correct signature: {:?}", signature);
+		assert!(Pair::verify(&signature, &message[..], &public));
+		assert!(!Pair::verify(&signature, "Other message", &public));
+	}
+
+	#[test]
+	fn generate_with_phrase_recovery_possible() {
+		let (pair1, phrase, _) = Pair::generate_with_phrase(None);
+		let (pair2, _) = Pair::from_phrase(&phrase, None).unwrap();
+
+		assert_eq!(pair1.public(), pair2.public());
+	}
+
+	#[test]
+	fn generate_with_password_phrase_recovery_possible() {
+		let (pair1, phrase, _) = Pair::generate_with_phrase(Some("password"));
+		let (pair2, _) = Pair::from_phrase(&phrase, Some("password")).unwrap();
+
+		assert_eq!(pair1.public(), pair2.public());
+	}
+
+	#[test]
+	fn password_does_something() {
+		let (pair1, phrase, _) = Pair::generate_with_phrase(Some("password"));
+		let (pair2, _) = Pair::from_phrase(&phrase, None).unwrap();
+
+		assert_ne!(pair1.public(), pair2.public());
+	}
+
+	#[test]
+	fn ss58check_roundtrip_works() {
+		let pair = Pair::from_seed(b"12345678901234567890123456789012");
+		let public = pair.public();
+		let s = public.to_ss58check();
+		println!("Correct: {}", s);
+		let cmp = Public::from_ss58check(&s).unwrap();
+		assert_eq!(cmp, public);
+	}
+}

substrate/primitives/core/src/ed25519.rs

@@ -0,0 +1,646 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+// tag::description[]
+//! Simple Ed25519 API.
+// end::description[]
+
+#[cfg(feature = "full_crypto")]
+use rstd::vec::Vec;
+
+use crate::{hash::H256, hash::H512};
+use codec::{Encode, Decode};
+
+#[cfg(feature = "full_crypto")]
+use blake2_rfc;
+#[cfg(feature = "std")]
+use substrate_bip39::seed_from_entropy;
+#[cfg(feature = "std")]
+use bip39::{Mnemonic, Language, MnemonicType};
+#[cfg(feature = "full_crypto")]
+use crate::crypto::{Pair as TraitPair, DeriveJunction, SecretStringError};
+#[cfg(feature = "std")]
+use crate::crypto::Ss58Codec;
+#[cfg(feature = "std")]
+use serde::{de, Serializer, Serialize, Deserializer, Deserialize};
+use crate::{crypto::{Public as TraitPublic, UncheckedFrom, CryptoType, Derive}};
+use runtime_interface::pass_by::PassByInner;
+use rstd::ops::Deref;
+
+/// A secret seed. It's not called a "secret key" because ring doesn't expose the secret keys
+/// of the key pair (yeah, dumb); as such we're forced to remember the seed manually if we
+/// will need it later (such as for HDKD).
+#[cfg(feature = "full_crypto")]
+type Seed = [u8; 32];
+
+/// A public key.
+#[cfg_attr(feature = "full_crypto", derive(Hash))]
+#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Encode, Decode, Default, PassByInner)]
+pub struct Public(pub [u8; 32]);
+
+/// A key pair.
+#[cfg(feature = "full_crypto")]
+pub struct Pair(ed25519_dalek::Keypair);
+
+#[cfg(feature = "full_crypto")]
+impl Clone for Pair {
+	fn clone(&self) -> Self {
+		Pair(ed25519_dalek::Keypair {
+			public: self.0.public.clone(),
+			secret: ed25519_dalek::SecretKey::from_bytes(self.0.secret.as_bytes())
+				.expect("key is always the correct size; qed")
+		})
+	}
+}
+
+impl AsRef<[u8; 32]> for Public {
+	fn as_ref(&self) -> &[u8; 32] {
+		&self.0
+	}
+}
+
+impl AsRef<[u8]> for Public {
+	fn as_ref(&self) -> &[u8] {
+		&self.0[..]
+	}
+}
+
+impl AsMut<[u8]> for Public {
+	fn as_mut(&mut self) -> &mut [u8] {
+		&mut self.0[..]
+	}
+}
+
+impl Deref for Public {
+    type Target = [u8];
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl rstd::convert::TryFrom<&[u8]> for Public {
+	type Error = ();
+
+	fn try_from(data: &[u8]) -> Result<Self, Self::Error> {
+		if data.len() == 32 {
+			let mut inner = [0u8; 32];
+			inner.copy_from_slice(data);
+			Ok(Public(inner))
+		} else {
+			Err(())
+		}
+	}
+}
+
+impl From<Public> for [u8; 32] {
+	fn from(x: Public) -> Self {
+		x.0
+	}
+}
+
+#[cfg(feature = "full_crypto")]
+impl From<Pair> for Public {
+	fn from(x: Pair) -> Self {
+		x.public()
+	}
+}
+
+impl From<Public> for H256 {
+	fn from(x: Public) -> Self {
+		x.0.into()
+	}
+}
+
+#[cfg(feature = "std")]
+impl std::str::FromStr for Public {
+    type Err = crate::crypto::PublicError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Self::from_ss58check(s)
+    }
+}
+
+impl UncheckedFrom<[u8; 32]> for Public {
+	fn unchecked_from(x: [u8; 32]) -> Self {
+		Public::from_raw(x)
+	}
+}
+
+impl UncheckedFrom<H256> for Public {
+	fn unchecked_from(x: H256) -> Self {
+		Public::from_h256(x)
+	}
+}
+
+#[cfg(feature = "std")]
+impl std::fmt::Display for Public {
+	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+		write!(f, "{}", self.to_ss58check())
+	}
+}
+
+impl rstd::fmt::Debug for Public {
+	#[cfg(feature = "std")]
+	fn fmt(&self, f: &mut rstd::fmt::Formatter) -> rstd::fmt::Result {
+		let s = self.to_ss58check();
+		write!(f, "{} ({}...)", crate::hexdisplay::HexDisplay::from(&self.0), &s[0..8])
+	}
+
+	#[cfg(not(feature = "std"))]
+	fn fmt(&self, _: &mut rstd::fmt::Formatter) -> rstd::fmt::Result {
+		Ok(())
+	}
+}
+
+#[cfg(feature = "std")]
+impl Serialize for Public {
+	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> where S: Serializer {
+		serializer.serialize_str(&self.to_ss58check())
+	}
+}
+
+#[cfg(feature = "std")]
+impl<'de> Deserialize<'de> for Public {
+	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> where D: Deserializer<'de> {
+		Public::from_ss58check(&String::deserialize(deserializer)?)
+			.map_err(|e| de::Error::custom(format!("{:?}", e)))
+	}
+}
+
+/// A signature (a 512-bit value).
+#[derive(Encode, Decode, PassByInner)]
+pub struct Signature(pub [u8; 64]);
+
+impl rstd::convert::TryFrom<&[u8]> for Signature {
+	type Error = ();
+
+	fn try_from(data: &[u8]) -> Result<Self, Self::Error> {
+		if data.len() == 64 {
+			let mut inner = [0u8; 64];
+			inner.copy_from_slice(data);
+			Ok(Signature(inner))
+		} else {
+			Err(())
+		}
+	}
+}
+
+impl Clone for Signature {
+	fn clone(&self) -> Self {
+		let mut r = [0u8; 64];
+		r.copy_from_slice(&self.0[..]);
+		Signature(r)
+	}
+}
+
+impl Default for Signature {
+	fn default() -> Self {
+		Signature([0u8; 64])
+	}
+}
+
+impl PartialEq for Signature {
+	fn eq(&self, b: &Self) -> bool {
+		self.0[..] == b.0[..]
+	}
+}
+
+impl Eq for Signature {}
+
+impl From<Signature> for H512 {
+	fn from(v: Signature) -> H512 {
+		H512::from(v.0)
+	}
+}
+
+impl From<Signature> for [u8; 64] {
+	fn from(v: Signature) -> [u8; 64] {
+		v.0
+	}
+}
+
+impl AsRef<[u8; 64]> for Signature {
+	fn as_ref(&self) -> &[u8; 64] {
+		&self.0
+	}
+}
+
+impl AsRef<[u8]> for Signature {
+	fn as_ref(&self) -> &[u8] {
+		&self.0[..]
+	}
+}
+
+impl AsMut<[u8]> for Signature {
+	fn as_mut(&mut self) -> &mut [u8] {
+		&mut self.0[..]
+	}
+}
+
+impl rstd::fmt::Debug for Signature {
+	#[cfg(feature = "std")]
+	fn fmt(&self, f: &mut rstd::fmt::Formatter) -> rstd::fmt::Result {
+		write!(f, "{}", crate::hexdisplay::HexDisplay::from(&self.0))
+	}
+
+	#[cfg(not(feature = "std"))]
+	fn fmt(&self, _: &mut rstd::fmt::Formatter) -> rstd::fmt::Result {
+		Ok(())
+	}
+}
+
+#[cfg(feature = "full_crypto")]
+impl rstd::hash::Hash for Signature {
+	fn hash<H: rstd::hash::Hasher>(&self, state: &mut H) {
+		rstd::hash::Hash::hash(&self.0[..], state);
+	}
+}
+
+impl Signature {
+	/// A new instance from the given 64-byte `data`.
+	///
+	/// NOTE: No checking goes on to ensure this is a real signature. Only use it if
+	/// you are certain that the array actually is a signature. GIGO!
+	pub fn from_raw(data: [u8; 64]) -> Signature {
+		Signature(data)
+	}
+
+	/// A new instance from the given slice that should be 64 bytes long.
+	///
+	/// NOTE: No checking goes on to ensure this is a real signature. Only use it if
+	/// you are certain that the array actually is a signature. GIGO!
+	pub fn from_slice(data: &[u8]) -> Self {
+		let mut r = [0u8; 64];
+		r.copy_from_slice(data);
+		Signature(r)
+	}
+
+	/// A new instance from an H512.
+	///
+	/// NOTE: No checking goes on to ensure this is a real signature. Only use it if
+	/// you are certain that the array actually is a signature. GIGO!
+	pub fn from_h512(v: H512) -> Signature {
+		Signature(v.into())
+	}
+}
+
+/// A localized signature also contains sender information.
+#[cfg(feature = "std")]
+#[derive(PartialEq, Eq, Clone, Debug, Encode, Decode)]
+pub struct LocalizedSignature {
+	/// The signer of the signature.
+	pub signer: Public,
+	/// The signature itself.
+	pub signature: Signature,
+}
+
+/// An error type for SS58 decoding.
+#[cfg(feature = "std")]
+#[derive(Clone, Copy, Eq, PartialEq, Debug)]
+pub enum PublicError {
+	/// Bad alphabet.
+	BadBase58,
+	/// Bad length.
+	BadLength,
+	/// Unknown version.
+	UnknownVersion,
+	/// Invalid checksum.
+	InvalidChecksum,
+}
+
+impl Public {
+	/// A new instance from the given 32-byte `data`.
+	///
+	/// NOTE: No checking goes on to ensure this is a real public key. Only use it if
+	/// you are certain that the array actually is a pubkey. GIGO!
+	pub fn from_raw(data: [u8; 32]) -> Self {
+		Public(data)
+	}
+
+	/// A new instance from an H256.
+	///
+	/// NOTE: No checking goes on to ensure this is a real public key. Only use it if
+	/// you are certain that the array actually is a pubkey. GIGO!
+	pub fn from_h256(x: H256) -> Self {
+		Public(x.into())
+	}
+
+	/// Return a slice filled with raw data.
+	pub fn as_array_ref(&self) -> &[u8; 32] {
+		self.as_ref()
+	}
+}
+
+impl TraitPublic for Public {
+	/// A new instance from the given slice that should be 32 bytes long.
+	///
+	/// NOTE: No checking goes on to ensure this is a real public key. Only use it if
+	/// you are certain that the array actually is a pubkey. GIGO!
+	fn from_slice(data: &[u8]) -> Self {
+		let mut r = [0u8; 32];
+		r.copy_from_slice(data);
+		Public(r)
+	}
+}
+
+impl Derive for Public {}
+
+/// Derive a single hard junction.
+#[cfg(feature = "full_crypto")]
+fn derive_hard_junction(secret_seed: &Seed, cc: &[u8; 32]) -> Seed {
+	("Ed25519HDKD", secret_seed, cc).using_encoded(|data| {
+		let mut res = [0u8; 32];
+		res.copy_from_slice(blake2_rfc::blake2b::blake2b(32, &[], data).as_bytes());
+		res
+	})
+}
+
+/// An error when deriving a key.
+#[cfg(feature = "full_crypto")]
+pub enum DeriveError {
+	/// A soft key was found in the path (and is unsupported).
+	SoftKeyInPath,
+}
+
+#[cfg(feature = "full_crypto")]
+impl TraitPair for Pair {
+	type Public = Public;
+	type Seed = Seed;
+	type Signature = Signature;
+	type DeriveError = DeriveError;
+
+	/// Generate new secure (random) key pair and provide the recovery phrase.
+	///
+	/// You can recover the same key later with `from_phrase`.
+	#[cfg(feature = "std")]
+	fn generate_with_phrase(password: Option<&str>) -> (Pair, String, Seed) {
+		let mnemonic = Mnemonic::new(MnemonicType::Words12, Language::English);
+		let phrase = mnemonic.phrase();
+		let (pair, seed) = Self::from_phrase(phrase, password)
+			.expect("All phrases generated by Mnemonic are valid; qed");
+		(
+			pair,
+			phrase.to_owned(),
+			seed,
+		)
+	}
+
+	/// Generate key pair from given recovery phrase and password.
+	#[cfg(feature = "std")]
+	fn from_phrase(phrase: &str, password: Option<&str>) -> Result<(Pair, Seed), SecretStringError> {
+		let big_seed = seed_from_entropy(
+			Mnemonic::from_phrase(phrase, Language::English)
+				.map_err(|_| SecretStringError::InvalidPhrase)?.entropy(),
+			password.unwrap_or(""),
+		).map_err(|_| SecretStringError::InvalidSeed)?;
+		let mut seed = Seed::default();
+		seed.copy_from_slice(&big_seed[0..32]);
+		Self::from_seed_slice(&big_seed[0..32]).map(|x| (x, seed))
+	}
+
+	/// Make a new key pair from secret seed material.
+	///
+	/// You should never need to use this; generate(), generate_with_phrasee
+	fn from_seed(seed: &Seed) -> Pair {
+		Self::from_seed_slice(&seed[..]).expect("seed has valid length; qed")
+	}
+
+	/// Make a new key pair from secret seed material. The slice must be 32 bytes long or it
+	/// will return `None`.
+	///
+	/// You should never need to use this; generate(), generate_with_phrase
+	fn from_seed_slice(seed_slice: &[u8]) -> Result<Pair, SecretStringError> {
+		let secret = ed25519_dalek::SecretKey::from_bytes(seed_slice)
+			.map_err(|_| SecretStringError::InvalidSeedLength)?;
+		let public = ed25519_dalek::PublicKey::from(secret.expand::<sha2::Sha512>());
+		Ok(Pair(ed25519_dalek::Keypair { secret, public }))
+	}
+
+	/// Derive a child key from a series of given junctions.
+	fn derive<Iter: Iterator<Item=DeriveJunction>>(&self,
+		path: Iter,
+		_seed: Option<Seed>,
+	) -> Result<(Pair, Option<Seed>), DeriveError> {
+		let mut acc = self.0.secret.to_bytes();
+		for j in path {
+			match j {
+				DeriveJunction::Soft(_cc) => return Err(DeriveError::SoftKeyInPath),
+				DeriveJunction::Hard(cc) => acc = derive_hard_junction(&acc, &cc),
+			}
+		}
+		Ok((Self::from_seed(&acc), Some(acc)))
+	}
+
+	/// Get the public key.
+	fn public(&self) -> Public {
+		let mut r = [0u8; 32];
+		let pk = self.0.public.as_bytes();
+		r.copy_from_slice(pk);
+		Public(r)
+	}
+
+	/// Sign a message.
+	fn sign(&self, message: &[u8]) -> Signature {
+		let r = self.0.sign::<sha2::Sha512>(message).to_bytes();
+		Signature::from_raw(r)
+	}
+
+	/// Verify a signature on a message. Returns true if the signature is good.
+	fn verify<M: AsRef<[u8]>>(sig: &Self::Signature, message: M, pubkey: &Self::Public) -> bool {
+		Self::verify_weak(&sig.0[..], message.as_ref(), pubkey)
+	}
+
+	/// Verify a signature on a message. Returns true if the signature is good.
+	///
+	/// This doesn't use the type system to ensure that `sig` and `pubkey` are the correct
+	/// size. Use it only if you're coming from byte buffers and need the speed.
+	fn verify_weak<P: AsRef<[u8]>, M: AsRef<[u8]>>(sig: &[u8], message: M, pubkey: P) -> bool {
+		let public_key = match ed25519_dalek::PublicKey::from_bytes(pubkey.as_ref()) {
+			Ok(pk) => pk,
+			Err(_) => return false,
+		};
+
+		let sig = match ed25519_dalek::Signature::from_bytes(sig) {
+			Ok(s) => s,
+			Err(_) => return false
+		};
+
+		match public_key.verify::<sha2::Sha512>(message.as_ref(), &sig) {
+			Ok(_) => true,
+			_ => false,
+		}
+	}
+
+	/// Return a vec filled with raw data.
+	fn to_raw_vec(&self) -> Vec<u8> {
+		self.seed().to_vec()
+	}
+}
+
+#[cfg(feature = "full_crypto")]
+impl Pair {
+	/// Get the seed for this key.
+	pub fn seed(&self) -> &Seed {
+		self.0.secret.as_bytes()
+	}
+
+	/// Exactly as `from_string` except that if no matches are found then, the the first 32
+	/// characters are taken (padded with spaces as necessary) and used as the MiniSecretKey.
+	#[cfg(feature = "std")]
+	pub fn from_legacy_string(s: &str, password_override: Option<&str>) -> Pair {
+		Self::from_string(s, password_override).unwrap_or_else(|_| {
+			let mut padded_seed: Seed = [' ' as u8; 32];
+			let len = s.len().min(32);
+			padded_seed[..len].copy_from_slice(&s.as_bytes()[..len]);
+			Self::from_seed(&padded_seed)
+		})
+	}
+}
+
+impl CryptoType for Public {
+	#[cfg(feature = "full_crypto")]
+	type Pair = Pair;
+}
+
+impl CryptoType for Signature {
+	#[cfg(feature = "full_crypto")]
+	type Pair = Pair;
+}
+
+#[cfg(feature = "full_crypto")]
+impl CryptoType for Pair {
+	type Pair = Pair;
+}
+
+#[cfg(test)]
+mod test {
+	use super::*;
+	use hex_literal::hex;
+	use crate::crypto::DEV_PHRASE;
+
+	#[test]
+	fn default_phrase_should_be_used() {
+		assert_eq!(
+			Pair::from_string("//Alice///password", None).unwrap().public(),
+			Pair::from_string(&format!("{}//Alice", DEV_PHRASE), Some("password")).unwrap().public(),
+		);
+	}
+
+	#[test]
+	fn seed_and_derive_should_work() {
+		let seed = hex!("9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60");
+		let pair = Pair::from_seed(&seed);
+		assert_eq!(pair.seed(), &seed);
+		let path = vec![DeriveJunction::Hard([0u8; 32])];
+		let derived = pair.derive(path.into_iter(), None).ok().unwrap().0;
+		assert_eq!(
+			derived.seed(),
+			&hex!("ede3354e133f9c8e337ddd6ee5415ed4b4ffe5fc7d21e933f4930a3730e5b21c")
+		);
+	}
+
+	#[test]
+	fn test_vector_should_work() {
+		let pair = Pair::from_seed(
+			&hex!("9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60")
+		);
+		let public = pair.public();
+		assert_eq!(public, Public::from_raw(
+			hex!("d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a")
+		));
+		let message = b"";
+		let signature = hex!("e5564300c360ac729086e2cc806e828a84877f1eb8e5d974d873e065224901555fb8821590a33bacc61e39701cf9b46bd25bf5f0595bbe24655141438e7a100b");
+		let signature = Signature::from_raw(signature);
+		assert!(&pair.sign(&message[..]) == &signature);
+		assert!(Pair::verify(&signature, &message[..], &public));
+	}
+
+	#[test]
+	fn test_vector_by_string_should_work() {
+		let pair = Pair::from_string(
+			"0x9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60",
+			None
+		).unwrap();
+		let public = pair.public();
+		assert_eq!(public, Public::from_raw(
+			hex!("d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a")
+		));
+		let message = b"";
+		let signature = hex!("e5564300c360ac729086e2cc806e828a84877f1eb8e5d974d873e065224901555fb8821590a33bacc61e39701cf9b46bd25bf5f0595bbe24655141438e7a100b");
+		let signature = Signature::from_raw(signature);
+		assert!(&pair.sign(&message[..]) == &signature);
+		assert!(Pair::verify(&signature, &message[..], &public));
+	}
+
+	#[test]
+	fn generated_pair_should_work() {
+		let (pair, _) = Pair::generate();
+		let public = pair.public();
+		let message = b"Something important";
+		let signature = pair.sign(&message[..]);
+		assert!(Pair::verify(&signature, &message[..], &public));
+		assert!(!Pair::verify(&signature, b"Something else", &public));
+	}
+
+	#[test]
+	fn seeded_pair_should_work() {
+		let pair = Pair::from_seed(b"12345678901234567890123456789012");
+		let public = pair.public();
+		assert_eq!(public, Public::from_raw(
+			hex!("2f8c6129d816cf51c374bc7f08c3e63ed156cf78aefb4a6550d97b87997977ee")
+		));
+		let message = hex!("2f8c6129d816cf51c374bc7f08c3e63ed156cf78aefb4a6550d97b87997977ee00000000000000000200d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a4500000000000000");
+		let signature = pair.sign(&message[..]);
+		println!("Correct signature: {:?}", signature);
+		assert!(Pair::verify(&signature, &message[..], &public));
+		assert!(!Pair::verify(&signature, "Other message", &public));
+	}
+
+	#[test]
+	fn generate_with_phrase_recovery_possible() {
+		let (pair1, phrase, _) = Pair::generate_with_phrase(None);
+		let (pair2, _) = Pair::from_phrase(&phrase, None).unwrap();
+
+		assert_eq!(pair1.public(), pair2.public());
+	}
+
+	#[test]
+	fn generate_with_password_phrase_recovery_possible() {
+		let (pair1, phrase, _) = Pair::generate_with_phrase(Some("password"));
+		let (pair2, _) = Pair::from_phrase(&phrase, Some("password")).unwrap();
+
+		assert_eq!(pair1.public(), pair2.public());
+	}
+
+	#[test]
+	fn password_does_something() {
+		let (pair1, phrase, _) = Pair::generate_with_phrase(Some("password"));
+		let (pair2, _) = Pair::from_phrase(&phrase, None).unwrap();
+
+		assert_ne!(pair1.public(), pair2.public());
+	}
+
+	#[test]
+	fn ss58check_roundtrip_works() {
+		let pair = Pair::from_seed(b"12345678901234567890123456789012");
+		let public = pair.public();
+		let s = public.to_ss58check();
+		println!("Correct: {}", s);
+		let cmp = Public::from_ss58check(&s).unwrap();
+		assert_eq!(cmp, public);
+	}
+}

substrate/primitives/core/src/hash.rs

@@ -0,0 +1,81 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! A fixed hash type.
+
+pub use primitive_types::{H160, H256, H512};
+
+/// Hash conversion. Used to convert between unbound associated hash types in traits,
+/// implemented by the same hash type.
+/// Panics if used to convert between different hash types.
+pub fn convert_hash<H1: Default + AsMut<[u8]>, H2: AsRef<[u8]>>(src: &H2) -> H1 {
+	let mut dest = H1::default();
+	assert_eq!(dest.as_mut().len(), src.as_ref().len());
+	dest.as_mut().copy_from_slice(src.as_ref());
+	dest
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use substrate_serializer as ser;
+
+	#[test]
+	fn test_h160() {
+		let tests = vec![
+			(Default::default(), "0x0000000000000000000000000000000000000000"),
+			(H160::from_low_u64_be(2), "0x0000000000000000000000000000000000000002"),
+			(H160::from_low_u64_be(15), "0x000000000000000000000000000000000000000f"),
+			(H160::from_low_u64_be(16), "0x0000000000000000000000000000000000000010"),
+			(H160::from_low_u64_be(1_000), "0x00000000000000000000000000000000000003e8"),
+			(H160::from_low_u64_be(100_000), "0x00000000000000000000000000000000000186a0"),
+			(H160::from_low_u64_be(u64::max_value()), "0x000000000000000000000000ffffffffffffffff"),
+		];
+
+		for (number, expected) in tests {
+			assert_eq!(format!("{:?}", expected), ser::to_string_pretty(&number));
+			assert_eq!(number, ser::from_str(&format!("{:?}", expected)).unwrap());
+		}
+	}
+
+	#[test]
+	fn test_h256() {
+		let tests = vec![
+			(Default::default(), "0x0000000000000000000000000000000000000000000000000000000000000000"),
+			(H256::from_low_u64_be(2), "0x0000000000000000000000000000000000000000000000000000000000000002"),
+			(H256::from_low_u64_be(15), "0x000000000000000000000000000000000000000000000000000000000000000f"),
+			(H256::from_low_u64_be(16), "0x0000000000000000000000000000000000000000000000000000000000000010"),
+			(H256::from_low_u64_be(1_000), "0x00000000000000000000000000000000000000000000000000000000000003e8"),
+			(H256::from_low_u64_be(100_000), "0x00000000000000000000000000000000000000000000000000000000000186a0"),
+			(H256::from_low_u64_be(u64::max_value()), "0x000000000000000000000000000000000000000000000000ffffffffffffffff"),
+		];
+
+		for (number, expected) in tests {
+			assert_eq!(format!("{:?}", expected), ser::to_string_pretty(&number));
+			assert_eq!(number, ser::from_str(&format!("{:?}", expected)).unwrap());
+		}
+	}
+
+	#[test]
+	fn test_invalid() {
+		assert!(ser::from_str::<H256>("\"0x000000000000000000000000000000000000000000000000000000000000000\"").unwrap_err().is_data());
+		assert!(ser::from_str::<H256>("\"0x000000000000000000000000000000000000000000000000000000000000000g\"").unwrap_err().is_data());
+		assert!(ser::from_str::<H256>("\"0x00000000000000000000000000000000000000000000000000000000000000000\"").unwrap_err().is_data());
+		assert!(ser::from_str::<H256>("\"\"").unwrap_err().is_data());
+		assert!(ser::from_str::<H256>("\"0\"").unwrap_err().is_data());
+		assert!(ser::from_str::<H256>("\"10\"").unwrap_err().is_data());
+	}
+}

substrate/primitives/core/src/hasher.rs

@@ -0,0 +1,53 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Substrate Blake2b Hasher implementation
+
+use hash_db::Hasher;
+use hash256_std_hasher::Hash256StdHasher;
+use crate::hash::H256;
+
+pub mod blake2 {
+	use super::{Hasher, Hash256StdHasher, H256};
+	#[cfg(feature = "std")]
+	use crate::hashing::blake2_256;
+
+	#[cfg(not(feature = "std"))]
+	extern "C" {
+		fn ext_blake2_256(data: *const u8, len: u32, out: *mut u8);
+	}
+	#[cfg(not(feature = "std"))]
+	fn blake2_256(data: &[u8]) -> [u8; 32] {
+		let mut result: [u8; 32] = Default::default();
+		unsafe {
+			ext_blake2_256(data.as_ptr(), data.len() as u32, result.as_mut_ptr());
+		}
+		result
+	}
+
+	/// Concrete implementation of Hasher using Blake2b 256-bit hashes
+	#[derive(Debug)]
+	pub struct Blake2Hasher;
+
+	impl Hasher for Blake2Hasher {
+		type Out = H256;
+		type StdHasher = Hash256StdHasher;
+		const LENGTH: usize = 32;
+		fn hash(x: &[u8]) -> Self::Out {
+			blake2_256(x).into()
+		}
+	}
+}

substrate/primitives/core/src/hashing.rs

@@ -0,0 +1,133 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Hashing functions.
+
+use blake2_rfc;
+use tiny_keccak::{Hasher, Keccak};
+use twox_hash;
+
+/// Do a Blake2 512-bit hash and place result in `dest`.
+pub fn blake2_512_into(data: &[u8], dest: &mut [u8; 64]) {
+	dest.copy_from_slice(blake2_rfc::blake2b::blake2b(64, &[], data).as_bytes());
+}
+
+/// Do a Blake2 512-bit hash and return result.
+pub fn blake2_512(data: &[u8]) -> [u8; 64] {
+	let mut r = [0; 64];
+	blake2_512_into(data, &mut r);
+	r
+}
+
+/// Do a Blake2 256-bit hash and place result in `dest`.
+pub fn blake2_256_into(data: &[u8], dest: &mut [u8; 32]) {
+	dest.copy_from_slice(blake2_rfc::blake2b::blake2b(32, &[], data).as_bytes());
+}
+
+/// Do a Blake2 256-bit hash and return result.
+pub fn blake2_256(data: &[u8]) -> [u8; 32] {
+	let mut r = [0; 32];
+	blake2_256_into(data, &mut r);
+	r
+}
+
+/// Do a Blake2 128-bit hash and place result in `dest`.
+pub fn blake2_128_into(data: &[u8], dest: &mut [u8; 16]) {
+	dest.copy_from_slice(blake2_rfc::blake2b::blake2b(16, &[], data).as_bytes());
+}
+
+/// Do a Blake2 128-bit hash and return result.
+pub fn blake2_128(data: &[u8]) -> [u8; 16] {
+	let mut r = [0; 16];
+	blake2_128_into(data, &mut r);
+	r
+}
+
+/// Do a XX 64-bit hash and place result in `dest`.
+pub fn twox_64_into(data: &[u8], dest: &mut [u8; 8]) {
+	use ::core::hash::Hasher;
+	let mut h0 = twox_hash::XxHash::with_seed(0);
+	h0.write(data);
+	let r0 = h0.finish();
+	use byteorder::{ByteOrder, LittleEndian};
+	LittleEndian::write_u64(&mut dest[0..8], r0);
+}
+
+/// Do a XX 64-bit hash and return result.
+pub fn twox_64(data: &[u8]) -> [u8; 8] {
+	let mut r: [u8; 8] = [0; 8];
+	twox_64_into(data, &mut r);
+	r
+}
+
+/// Do a XX 128-bit hash and place result in `dest`.
+pub fn twox_128_into(data: &[u8], dest: &mut [u8; 16]) {
+	use ::core::hash::Hasher;
+	let mut h0 = twox_hash::XxHash::with_seed(0);
+	let mut h1 = twox_hash::XxHash::with_seed(1);
+	h0.write(data);
+	h1.write(data);
+	let r0 = h0.finish();
+	let r1 = h1.finish();
+	use byteorder::{ByteOrder, LittleEndian};
+	LittleEndian::write_u64(&mut dest[0..8], r0);
+	LittleEndian::write_u64(&mut dest[8..16], r1);
+}
+
+/// Do a XX 128-bit hash and return result.
+pub fn twox_128(data: &[u8]) -> [u8; 16] {
+	let mut r: [u8; 16] = [0; 16];
+	twox_128_into(data, &mut r);
+	r
+}
+
+/// Do a XX 256-bit hash and place result in `dest`.
+pub fn twox_256_into(data: &[u8], dest: &mut [u8; 32]) {
+	use ::core::hash::Hasher;
+	use byteorder::{ByteOrder, LittleEndian};
+	let mut h0 = twox_hash::XxHash::with_seed(0);
+	let mut h1 = twox_hash::XxHash::with_seed(1);
+	let mut h2 = twox_hash::XxHash::with_seed(2);
+	let mut h3 = twox_hash::XxHash::with_seed(3);
+	h0.write(data);
+	h1.write(data);
+	h2.write(data);
+	h3.write(data);
+	let r0 = h0.finish();
+	let r1 = h1.finish();
+	let r2 = h2.finish();
+	let r3 = h3.finish();
+	LittleEndian::write_u64(&mut dest[0..8], r0);
+	LittleEndian::write_u64(&mut dest[8..16], r1);
+	LittleEndian::write_u64(&mut dest[16..24], r2);
+	LittleEndian::write_u64(&mut dest[24..32], r3);
+}
+
+/// Do a XX 256-bit hash and return result.
+pub fn twox_256(data: &[u8]) -> [u8; 32] {
+	let mut r: [u8; 32] = [0; 32];
+	twox_256_into(data, &mut r);
+	r
+}
+
+/// Do a keccak 256 hash and return result.
+pub fn keccak_256(data: &[u8]) -> [u8; 32] {
+	let mut keccak = Keccak::v256();
+	keccak.update(data);
+	let mut output = [0u8; 32];
+	keccak.finalize(&mut output);
+	output
+}

substrate/primitives/core/src/hexdisplay.rs

@@ -0,0 +1,102 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Wrapper type for byte collections that outputs hex.
+
+/// Simple wrapper to display hex representation of bytes.
+pub struct HexDisplay<'a>(&'a [u8]);
+
+impl<'a> HexDisplay<'a> {
+	/// Create new instance that will display `d` as a hex string when displayed.
+	pub fn from<R: AsBytesRef>(d: &'a R) -> Self { HexDisplay(d.as_bytes_ref()) }
+}
+
+impl<'a> rstd::fmt::Display for HexDisplay<'a> {
+	fn fmt(&self, f: &mut rstd::fmt::Formatter) -> Result<(), rstd::fmt::Error> {
+		if self.0.len() < 1027 {
+			for byte in self.0 {
+				f.write_fmt(format_args!("{:02x}", byte))?;
+			}
+		} else {
+			for byte in &self.0[0..512] {
+				f.write_fmt(format_args!("{:02x}", byte))?;
+			}
+			f.write_str("...")?;
+			for byte in &self.0[self.0.len() - 512..] {
+				f.write_fmt(format_args!("{:02x}", byte))?;
+			}
+		}
+		Ok(())
+	}
+}
+
+impl<'a> rstd::fmt::Debug for HexDisplay<'a> {
+	fn fmt(&self, f: &mut rstd::fmt::Formatter) -> Result<(), rstd::fmt::Error> {
+		for byte in self.0 {
+			f.write_fmt(format_args!("{:02x}", byte))?;
+		}
+		Ok(())
+	}
+}
+
+/// Simple trait to transform various types to `&[u8]`
+pub trait AsBytesRef {
+	/// Transform `self` into `&[u8]`.
+	fn as_bytes_ref(&self) -> &[u8];
+}
+
+impl<'a> AsBytesRef for &'a [u8] {
+	fn as_bytes_ref(&self) -> &[u8] { self }
+}
+
+impl AsBytesRef for [u8] {
+	fn as_bytes_ref(&self) -> &[u8] { &self }
+}
+
+impl AsBytesRef for Vec<u8> {
+	fn as_bytes_ref(&self) -> &[u8] { &self }
+}
+
+macro_rules! impl_non_endians {
+	( $( $t:ty ),* ) => { $(
+		impl AsBytesRef for $t {
+			fn as_bytes_ref(&self) -> &[u8] { &self[..] }
+		}
+	)* }
+}
+
+impl_non_endians!([u8; 1], [u8; 2], [u8; 3], [u8; 4], [u8; 5], [u8; 6], [u8; 7], [u8; 8],
+	[u8; 10], [u8; 12], [u8; 14], [u8; 16], [u8; 20], [u8; 24], [u8; 28], [u8; 32], [u8; 40],
+	[u8; 48], [u8; 56], [u8; 64], [u8; 65], [u8; 80], [u8; 96], [u8; 112], [u8; 128]);
+
+/// Format into ASCII + # + hex, suitable for storage key preimages.
+pub fn ascii_format(asciish: &[u8]) -> String {
+	let mut r = String::new();
+	let mut latch = false;
+	for c in asciish {
+		match (latch, *c) {
+			(false, 32..=127) => r.push(*c as char),
+			_ => {
+				if !latch {
+					r.push('#');
+					latch = true;
+				}
+				r.push_str(&format!("{:02x}", *c));
+			}
+		}
+	}
+	r
+}

substrate/primitives/core/src/lib.rs

@@ -0,0 +1,311 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Shareable Substrate types.
+
+#![warn(missing_docs)]
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+/// Initialize a key-value collection from array.
+///
+/// Creates a vector of given pairs and calls `collect` on the iterator from it.
+/// Can be used to create a `HashMap`.
+#[macro_export]
+macro_rules! map {
+	($( $name:expr => $value:expr ),* $(,)? ) => (
+		vec![ $( ( $name, $value ) ),* ].into_iter().collect()
+	);
+}
+
+use rstd::prelude::*;
+use rstd::ops::Deref;
+#[cfg(feature = "std")]
+use std::borrow::Cow;
+#[cfg(feature = "std")]
+use serde::{Serialize, Deserialize};
+#[cfg(feature = "std")]
+pub use serde;
+#[doc(hidden)]
+pub use codec::{Encode, Decode};
+
+pub use substrate_debug_derive::RuntimeDebug;
+
+#[cfg(feature = "std")]
+pub use impl_serde::serialize as bytes;
+
+#[cfg(feature = "full_crypto")]
+pub mod hashing;
+#[cfg(feature = "full_crypto")]
+pub use hashing::{blake2_128, blake2_256, twox_64, twox_128, twox_256, keccak_256};
+#[cfg(feature = "std")]
+pub mod hexdisplay;
+pub mod crypto;
+
+pub mod u32_trait;
+
+pub mod ed25519;
+pub mod sr25519;
+pub mod ecdsa;
+pub mod hash;
+mod hasher;
+pub mod offchain;
+pub mod sandbox;
+pub mod uint;
+mod changes_trie;
+#[cfg(feature = "std")]
+pub mod traits;
+pub mod testing;
+
+#[cfg(test)]
+mod tests;
+
+pub use self::hash::{H160, H256, H512, convert_hash};
+pub use self::uint::U256;
+pub use changes_trie::ChangesTrieConfiguration;
+#[cfg(feature = "full_crypto")]
+pub use crypto::{DeriveJunction, Pair, Public};
+
+pub use hash_db::Hasher;
+// Switch back to Blake after PoC-3 is out
+// pub use self::hasher::blake::BlakeHasher;
+pub use self::hasher::blake2::Blake2Hasher;
+
+pub use primitives_storage as storage;
+
+#[doc(hidden)]
+pub use rstd;
+
+/// Context for executing a call into the runtime.
+pub enum ExecutionContext {
+	/// Context for general importing (including own blocks).
+	Importing,
+	/// Context used when syncing the blockchain.
+	Syncing,
+	/// Context used for block construction.
+	BlockConstruction,
+	/// Context used for offchain calls.
+	///
+	/// This allows passing offchain extension and customizing available capabilities.
+	OffchainCall(Option<(Box<dyn offchain::Externalities>, offchain::Capabilities)>),
+}
+
+impl ExecutionContext {
+	/// Returns the capabilities of particular context.
+	pub fn capabilities(&self) -> offchain::Capabilities {
+		use ExecutionContext::*;
+
+		match self {
+			Importing | Syncing | BlockConstruction =>
+				offchain::Capabilities::none(),
+			// Enable keystore by default for offchain calls. CC @bkchr
+			OffchainCall(None) => [offchain::Capability::Keystore][..].into(),
+			OffchainCall(Some((_, capabilities))) => *capabilities,
+		}
+	}
+}
+
+/// Hex-serialized shim for `Vec<u8>`.
+#[derive(PartialEq, Eq, Clone, RuntimeDebug)]
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, PartialOrd, Ord))]
+pub struct Bytes(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
+
+impl From<Vec<u8>> for Bytes {
+	fn from(s: Vec<u8>) -> Self { Bytes(s) }
+}
+
+impl From<OpaqueMetadata> for Bytes {
+	fn from(s: OpaqueMetadata) -> Self { Bytes(s.0) }
+}
+
+impl Deref for Bytes {
+	type Target = [u8];
+	fn deref(&self) -> &[u8] { &self.0[..] }
+}
+
+/// Stores the encoded `RuntimeMetadata` for the native side as opaque type.
+#[derive(Encode, Decode, PartialEq)]
+pub struct OpaqueMetadata(Vec<u8>);
+
+impl OpaqueMetadata {
+	/// Creates a new instance with the given metadata blob.
+	pub fn new(metadata: Vec<u8>) -> Self {
+		OpaqueMetadata(metadata)
+	}
+}
+
+impl rstd::ops::Deref for OpaqueMetadata {
+	type Target = Vec<u8>;
+
+	fn deref(&self) -> &Self::Target {
+		&self.0
+	}
+}
+
+/// Something that is either a native or an encoded value.
+#[cfg(feature = "std")]
+pub enum NativeOrEncoded<R> {
+	/// The native representation.
+	Native(R),
+	/// The encoded representation.
+	Encoded(Vec<u8>)
+}
+
+#[cfg(feature = "std")]
+impl<R: codec::Encode> rstd::fmt::Debug for NativeOrEncoded<R> {
+	fn fmt(&self, f: &mut rstd::fmt::Formatter) -> rstd::fmt::Result {
+		hexdisplay::HexDisplay::from(&self.as_encoded().as_ref()).fmt(f)
+	}
+}
+
+#[cfg(feature = "std")]
+impl<R: codec::Encode> NativeOrEncoded<R> {
+	/// Return the value as the encoded format.
+	pub fn as_encoded(&self) -> Cow<'_, [u8]> {
+		match self {
+			NativeOrEncoded::Encoded(e) => Cow::Borrowed(e.as_slice()),
+			NativeOrEncoded::Native(n) => Cow::Owned(n.encode()),
+		}
+	}
+
+	/// Return the value as the encoded format.
+	pub fn into_encoded(self) -> Vec<u8> {
+		match self {
+			NativeOrEncoded::Encoded(e) => e,
+			NativeOrEncoded::Native(n) => n.encode(),
+		}
+	}
+}
+
+#[cfg(feature = "std")]
+impl<R: PartialEq + codec::Decode> PartialEq for NativeOrEncoded<R> {
+	fn eq(&self, other: &Self) -> bool {
+		match (self, other) {
+			(NativeOrEncoded::Native(l), NativeOrEncoded::Native(r)) => l == r,
+			(NativeOrEncoded::Native(n), NativeOrEncoded::Encoded(e)) |
+			(NativeOrEncoded::Encoded(e), NativeOrEncoded::Native(n)) =>
+				Some(n) == codec::Decode::decode(&mut &e[..]).ok().as_ref(),
+			(NativeOrEncoded::Encoded(l), NativeOrEncoded::Encoded(r)) => l == r,
+		}
+	}
+}
+
+/// A value that is never in a native representation.
+/// This is type is useful in conjuction with `NativeOrEncoded`.
+#[cfg(feature = "std")]
+#[derive(PartialEq)]
+pub enum NeverNativeValue {}
+
+#[cfg(feature = "std")]
+impl codec::Encode for NeverNativeValue {
+	fn encode(&self) -> Vec<u8> {
+		// The enum is not constructable, so this function should never be callable!
+		unreachable!()
+	}
+}
+
+#[cfg(feature = "std")]
+impl codec::EncodeLike for NeverNativeValue {}
+
+#[cfg(feature = "std")]
+impl codec::Decode for NeverNativeValue {
+	fn decode<I: codec::Input>(_: &mut I) -> Result<Self, codec::Error> {
+		Err("`NeverNativeValue` should never be decoded".into())
+	}
+}
+
+/// Provide a simple 4 byte identifier for a type.
+pub trait TypeId {
+	/// Simple 4 byte identifier.
+	const TYPE_ID: [u8; 4];
+}
+
+/// A log level matching the one from `log` crate.
+///
+/// Used internally by `runtime_io::log` method.
+#[derive(Encode, Decode, runtime_interface::pass_by::PassByEnum, Copy, Clone)]
+pub enum LogLevel {
+	/// `Error` log level.
+	Error = 1,
+	/// `Warn` log level.
+	Warn = 2,
+	/// `Info` log level.
+	Info = 3,
+	/// `Debug` log level.
+	Debug = 4,
+	/// `Trace` log level.
+	Trace = 5,
+}
+
+impl From<u32> for LogLevel {
+	fn from(val: u32) -> Self {
+		match val {
+			x if x == LogLevel::Warn as u32 => LogLevel::Warn,
+			x if x == LogLevel::Info as u32 => LogLevel::Info,
+			x if x == LogLevel::Debug as u32 => LogLevel::Debug,
+			x if x == LogLevel::Trace as u32 => LogLevel::Trace,
+			_ => LogLevel::Error,
+		}
+	}
+}
+
+impl From<log::Level> for LogLevel {
+	fn from(l: log::Level) -> Self {
+		use log::Level::*;
+		match l {
+			Error => Self::Error,
+			Warn => Self::Warn,
+			Info => Self::Info,
+			Debug => Self::Debug,
+			Trace => Self::Trace,
+		}
+	}
+}
+
+impl From<LogLevel> for log::Level {
+	fn from(l: LogLevel) -> Self {
+		use self::LogLevel::*;
+		match l {
+			Error => Self::Error,
+			Warn => Self::Warn,
+			Info => Self::Info,
+			Debug => Self::Debug,
+			Trace => Self::Trace,
+		}
+	}
+}
+
+/// Encodes the given value into a buffer and returns the pointer and the length as a single `u64`.
+///
+/// When Substrate calls into Wasm it expects a fixed signature for functions exported
+/// from the Wasm blob. The return value of this signature is always a `u64`.
+/// This `u64` stores the pointer to the encoded return value and the length of this encoded value.
+/// The low `32bits` are reserved for the pointer, followed by `32bit` for the length.
+#[cfg(not(feature = "std"))]
+pub fn to_substrate_wasm_fn_return_value(value: &impl Encode) -> u64 {
+	let encoded = value.encode();
+
+	let ptr = encoded.as_ptr() as u64;
+	let length = encoded.len() as u64;
+	let res = ptr | (length << 32);
+
+	// Leak the output vector to avoid it being freed.
+	// This is fine in a WASM context since the heap
+	// will be discarded after the call.
+	rstd::mem::forget(encoded);
+
+	res
+}

substrate/primitives/core/src/offchain.rs

@@ -0,0 +1,698 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Offchain workers types
+
+use codec::{Encode, Decode};
+use rstd::{prelude::{Vec, Box}, convert::TryFrom};
+use crate::RuntimeDebug;
+use runtime_interface::pass_by::{PassByCodec, PassByInner, PassByEnum};
+
+pub use crate::crypto::KeyTypeId;
+
+/// A type of supported crypto.
+#[derive(Clone, Copy, PartialEq, Eq, Encode, Decode, RuntimeDebug, PassByEnum)]
+#[repr(C)]
+pub enum StorageKind {
+	/// Persistent storage is non-revertible and not fork-aware. It means that any value
+	/// set by the offchain worker triggered at block `N(hash1)` is persisted even
+	/// if that block is reverted as non-canonical and is available for the worker
+	/// that is re-run at block `N(hash2)`.
+	/// This storage can be used by offchain workers to handle forks
+	/// and coordinate offchain workers running on different forks.
+	PERSISTENT = 1,
+	/// Local storage is revertible and fork-aware. It means that any value
+	/// set by the offchain worker triggered at block `N(hash1)` is reverted
+	/// if that block is reverted as non-canonical and is NOT available for the worker
+	/// that is re-run at block `N(hash2)`.
+	LOCAL = 2,
+}
+
+impl TryFrom<u32> for StorageKind {
+	type Error = ();
+
+	fn try_from(kind: u32) -> Result<Self, Self::Error> {
+		match kind {
+			e if e == u32::from(StorageKind::PERSISTENT as u8) => Ok(StorageKind::PERSISTENT),
+			e if e == u32::from(StorageKind::LOCAL as u8) => Ok(StorageKind::LOCAL),
+			_ => Err(()),
+		}
+	}
+}
+
+impl From<StorageKind> for u32 {
+	fn from(c: StorageKind) -> Self {
+		c as u8 as u32
+	}
+}
+
+/// Opaque type for offchain http requests.
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, RuntimeDebug, Encode, Decode, PassByInner)]
+#[cfg_attr(feature = "std", derive(Hash))]
+pub struct HttpRequestId(pub u16);
+
+impl From<HttpRequestId> for u32 {
+	fn from(c: HttpRequestId) -> Self {
+		c.0 as u32
+	}
+}
+
+/// An error enum returned by some http methods.
+#[derive(Clone, Copy, PartialEq, Eq, RuntimeDebug, Encode, Decode, PassByEnum)]
+#[repr(C)]
+pub enum HttpError {
+	/// The requested action couldn't been completed within a deadline.
+	DeadlineReached = 1,
+	/// There was an IO Error while processing the request.
+	IoError = 2,
+	/// The ID of the request is invalid in this context.
+	Invalid = 3,
+}
+
+impl TryFrom<u32> for HttpError {
+	type Error = ();
+
+	fn try_from(error: u32) -> Result<Self, Self::Error> {
+		match error {
+			e if e == HttpError::DeadlineReached as u8 as u32 => Ok(HttpError::DeadlineReached),
+			e if e == HttpError::IoError as u8 as u32 => Ok(HttpError::IoError),
+			e if e == HttpError::Invalid as u8 as u32 => Ok(HttpError::Invalid),
+			_ => Err(())
+		}
+	}
+}
+
+impl From<HttpError> for u32 {
+	fn from(c: HttpError) -> Self {
+		c as u8 as u32
+	}
+}
+
+/// Status of the HTTP request
+#[derive(Clone, Copy, PartialEq, Eq, RuntimeDebug, Encode, Decode, PassByCodec)]
+pub enum HttpRequestStatus {
+	/// Deadline was reached while we waited for this request to finish.
+	///
+	/// Note the deadline is controlled by the calling part, it not necessarily
+	/// means that the request has timed out.
+	DeadlineReached,
+	/// An error has occured during the request, for example a timeout or the
+	/// remote has closed our socket.
+	///
+	/// The request is now considered destroyed. To retry the request you need
+	/// to construct it again.
+	IoError,
+	/// The passed ID is invalid in this context.
+	Invalid,
+	/// The request has finished with given status code.
+	Finished(u16),
+}
+
+impl From<HttpRequestStatus> for u32 {
+	fn from(status: HttpRequestStatus) -> Self {
+		match status {
+			HttpRequestStatus::Invalid => 0,
+			HttpRequestStatus::DeadlineReached => 10,
+			HttpRequestStatus::IoError => 20,
+			HttpRequestStatus::Finished(code) => u32::from(code),
+		}
+	}
+}
+
+impl TryFrom<u32> for HttpRequestStatus {
+	type Error = ();
+
+	fn try_from(status: u32) -> Result<Self, Self::Error> {
+		match status {
+			0 => Ok(HttpRequestStatus::Invalid),
+			10 => Ok(HttpRequestStatus::DeadlineReached),
+			20 => Ok(HttpRequestStatus::IoError),
+			100..=999 => u16::try_from(status).map(HttpRequestStatus::Finished).map_err(|_| ()),
+			_ => Err(()),
+		}
+	}
+}
+
+/// A blob to hold information about the local node's network state
+/// without committing to its format.
+#[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug, PassByCodec)]
+pub struct OpaqueNetworkState {
+	/// PeerId of the local node.
+	pub peer_id: OpaquePeerId,
+	/// List of addresses the node knows it can be reached as.
+	pub external_addresses: Vec<OpaqueMultiaddr>,
+}
+
+/// Simple blob to hold a `PeerId` without committing to its format.
+#[derive(Default, Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug, PassByInner)]
+pub struct OpaquePeerId(pub Vec<u8>);
+
+impl OpaquePeerId {
+	/// Create new `OpaquePeerId`
+	pub fn new(vec: Vec<u8>) -> Self {
+		OpaquePeerId(vec)
+	}
+}
+
+/// Simple blob to hold a `Multiaddr` without committing to its format.
+#[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug, PassByInner)]
+pub struct OpaqueMultiaddr(pub Vec<u8>);
+
+impl OpaqueMultiaddr {
+	/// Create new `OpaqueMultiaddr`
+	pub fn new(vec: Vec<u8>) -> Self {
+		OpaqueMultiaddr(vec)
+	}
+}
+
+/// Opaque timestamp type
+#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default, RuntimeDebug, PassByInner, Encode, Decode)]
+pub struct Timestamp(u64);
+
+/// Duration type
+#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default, RuntimeDebug, PassByInner, Encode, Decode)]
+pub struct Duration(u64);
+
+impl Duration {
+	/// Create new duration representing given number of milliseconds.
+	pub fn from_millis(millis: u64) -> Self {
+		Duration(millis)
+	}
+
+	/// Returns number of milliseconds this Duration represents.
+	pub fn millis(&self) -> u64 {
+		self.0
+	}
+}
+
+impl Timestamp {
+	/// Creates new `Timestamp` given unix timestamp in miliseconds.
+	pub fn from_unix_millis(millis: u64) -> Self {
+		Timestamp(millis)
+	}
+
+	/// Increase the timestamp by given `Duration`.
+	pub fn add(&self, duration: Duration) -> Timestamp {
+		Timestamp(self.0.saturating_add(duration.0))
+	}
+
+	/// Decrease the timestamp by given `Duration`
+	pub fn sub(&self, duration: Duration) -> Timestamp {
+		Timestamp(self.0.saturating_sub(duration.0))
+	}
+
+	/// Returns a saturated difference (Duration) between two Timestamps.
+	pub fn diff(&self, other: &Self) -> Duration {
+		Duration(self.0.saturating_sub(other.0))
+	}
+
+	/// Return number of milliseconds since UNIX epoch.
+	pub fn unix_millis(&self) -> u64 {
+		self.0
+	}
+}
+
+/// Execution context extra capabilities.
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+#[repr(u8)]
+pub enum Capability {
+	/// Access to transaction pool.
+	TransactionPool = 1,
+	/// External http calls.
+	Http = 2,
+	/// Keystore access.
+	Keystore = 4,
+	/// Randomness source.
+	Randomness = 8,
+	/// Access to opaque network state.
+	NetworkState = 16,
+	/// Access to offchain worker DB (read only).
+	OffchainWorkerDbRead = 32,
+	/// Access to offchain worker DB (writes).
+	OffchainWorkerDbWrite = 64,
+}
+
+/// A set of capabilities
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub struct Capabilities(u8);
+
+impl Capabilities {
+	/// Return an object representing an empty set of capabilities.
+	pub fn none() -> Self {
+		Self(0)
+	}
+
+	/// Return an object representing all capabilities enabled.
+	pub fn all() -> Self {
+		Self(u8::max_value())
+	}
+
+	/// Return capabilities for rich offchain calls.
+	///
+	/// Those calls should be allowed to sign and submit transactions
+	/// and access offchain workers database (but read only!).
+	pub fn rich_offchain_call() -> Self {
+		[
+			Capability::TransactionPool,
+			Capability::Keystore,
+			Capability::OffchainWorkerDbRead,
+		][..].into()
+	}
+
+	/// Check if particular capability is enabled.
+	pub fn has(&self, capability: Capability) -> bool {
+		self.0 & capability as u8 != 0
+	}
+
+	/// Check if this capability object represents all capabilities.
+	pub fn has_all(&self) -> bool {
+		self == &Capabilities::all()
+	}
+}
+
+impl<'a> From<&'a [Capability]> for Capabilities {
+	fn from(list: &'a [Capability]) -> Self {
+		Capabilities(list.iter().fold(0_u8, |a, b| a | *b as u8))
+	}
+}
+
+/// An extended externalities for offchain workers.
+pub trait Externalities: Send {
+	/// Returns if the local node is a potential validator.
+	///
+	/// Even if this function returns `true`, it does not mean that any keys are configured
+	/// and that the validator is registered in the chain.
+	fn is_validator(&self) -> bool;
+	/// Submit transaction.
+	///
+	/// The transaction will end up in the pool and be propagated to others.
+	fn submit_transaction(&mut self, extrinsic: Vec<u8>) -> Result<(), ()>;
+
+	/// Returns information about the local node's network state.
+	fn network_state(&self) -> Result<OpaqueNetworkState, ()>;
+
+	/// Returns current UNIX timestamp (in millis)
+	fn timestamp(&mut self) -> Timestamp;
+
+	/// Pause the execution until `deadline` is reached.
+	fn sleep_until(&mut self, deadline: Timestamp);
+
+	/// Returns a random seed.
+	///
+	/// This is a trully random non deterministic seed generated by host environment.
+	/// Obviously fine in the off-chain worker context.
+	fn random_seed(&mut self) -> [u8; 32];
+
+	/// Sets a value in the local storage.
+	///
+	/// Note this storage is not part of the consensus, it's only accessible by
+	/// offchain worker tasks running on the same machine. It IS persisted between runs.
+	fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]);
+
+	/// Sets a value in the local storage if it matches current value.
+	///
+	/// Since multiple offchain workers may be running concurrently, to prevent
+	/// data races use CAS to coordinate between them.
+	///
+	/// Returns `true` if the value has been set, `false` otherwise.
+	///
+	/// Note this storage is not part of the consensus, it's only accessible by
+	/// offchain worker tasks running on the same machine. It IS persisted between runs.
+	fn local_storage_compare_and_set(
+		&mut self,
+		kind: StorageKind,
+		key: &[u8],
+		old_value: Option<&[u8]>,
+		new_value: &[u8],
+	) -> bool;
+
+	/// Gets a value from the local storage.
+	///
+	/// If the value does not exist in the storage `None` will be returned.
+	/// Note this storage is not part of the consensus, it's only accessible by
+	/// offchain worker tasks running on the same machine. It IS persisted between runs.
+	fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option<Vec<u8>>;
+
+	/// Initiates a http request given HTTP verb and the URL.
+	///
+	/// Meta is a future-reserved field containing additional, parity-scale-codec encoded parameters.
+	/// Returns the id of newly started request.
+	///
+	/// Returns an error if:
+	/// - No new request identifier could be allocated.
+	/// - The method or URI contain invalid characters.
+	///
+	fn http_request_start(
+		&mut self,
+		method: &str,
+		uri: &str,
+		meta: &[u8]
+	) -> Result<HttpRequestId, ()>;
+
+	/// Append header to the request.
+	///
+	/// Calling this function multiple times with the same header name continues appending new
+	/// headers. In other words, headers are never replaced.
+	///
+	/// Returns an error if:
+	/// - The request identifier is invalid.
+	/// - You have called `http_request_write_body` on that request.
+	/// - The name or value contain invalid characters.
+	///
+	/// An error doesn't poison the request, and you can continue as if the call had never been
+	/// made.
+	///
+	fn http_request_add_header(
+		&mut self,
+		request_id: HttpRequestId,
+		name: &str,
+		value: &str
+	) -> Result<(), ()>;
+
+	/// Write a chunk of request body.
+	///
+	/// Calling this function with a non-empty slice may or may not start the
+	/// HTTP request. Calling this function with an empty chunks finalizes the
+	/// request and always starts it. It is no longer valid to write more data
+	/// afterwards.
+	/// Passing `None` as deadline blocks forever.
+	///
+	/// Returns an error if:
+	/// - The request identifier is invalid.
+	/// - `http_response_wait` has already been called on this request.
+	/// - The deadline is reached.
+	/// - An I/O error has happened, for example the remote has closed our
+	///   request. The request is then considered invalid.
+	///
+	fn http_request_write_body(
+		&mut self,
+		request_id: HttpRequestId,
+		chunk: &[u8],
+		deadline: Option<Timestamp>
+	) -> Result<(), HttpError>;
+
+	/// Block and wait for the responses for given requests.
+	///
+	/// Returns a vector of request statuses (the len is the same as ids).
+	/// Note that if deadline is not provided the method will block indefinitely,
+	/// otherwise unready responses will produce `DeadlineReached` status.
+	///
+	/// If a response returns an `IoError`, it is then considered destroyed.
+	/// Its id is then invalid.
+	///
+	/// Passing `None` as deadline blocks forever.
+	fn http_response_wait(
+		&mut self,
+		ids: &[HttpRequestId],
+		deadline: Option<Timestamp>
+	) -> Vec<HttpRequestStatus>;
+
+	/// Read all response headers.
+	///
+	/// Returns a vector of pairs `(HeaderKey, HeaderValue)`.
+	///
+	/// Dispatches the request if it hasn't been done yet. It is no longer
+	/// valid to modify the headers or write data to the request.
+	///
+	/// Returns an empty list if the identifier is unknown/invalid, hasn't
+	/// received a response, or has finished.
+	fn http_response_headers(
+		&mut self,
+		request_id: HttpRequestId
+	) -> Vec<(Vec<u8>, Vec<u8>)>;
+
+	/// Read a chunk of body response to given buffer.
+	///
+	/// Dispatches the request if it hasn't been done yet. It is no longer
+	/// valid to modify the headers or write data to the request.
+	///
+	/// Returns the number of bytes written or an error in case a deadline
+	/// is reached or server closed the connection.
+	/// Passing `None` as a deadline blocks forever.
+	///
+	/// If `Ok(0)` or `Err(IoError)` is returned, the request is considered
+	/// destroyed. Doing another read or getting the response's headers, for
+	/// example, is then invalid.
+	///
+	/// Returns an error if:
+	/// - The request identifier is invalid.
+	/// - The deadline is reached.
+	/// - An I/O error has happened, for example the remote has closed our
+	///   request. The request is then considered invalid.
+	///
+	fn http_response_read_body(
+		&mut self,
+		request_id: HttpRequestId,
+		buffer: &mut [u8],
+		deadline: Option<Timestamp>
+	) -> Result<usize, HttpError>;
+
+}
+impl<T: Externalities + ?Sized> Externalities for Box<T> {
+	fn is_validator(&self) -> bool {
+		(& **self).is_validator()
+	}
+
+	fn submit_transaction(&mut self, ex: Vec<u8>) -> Result<(), ()> {
+		(&mut **self).submit_transaction(ex)
+	}
+
+	fn network_state(&self) -> Result<OpaqueNetworkState, ()> {
+		(& **self).network_state()
+	}
+
+	fn timestamp(&mut self) -> Timestamp {
+		(&mut **self).timestamp()
+	}
+
+	fn sleep_until(&mut self, deadline: Timestamp) {
+		(&mut **self).sleep_until(deadline)
+	}
+
+	fn random_seed(&mut self) -> [u8; 32] {
+		(&mut **self).random_seed()
+	}
+
+	fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) {
+		(&mut **self).local_storage_set(kind, key, value)
+	}
+
+	fn local_storage_compare_and_set(
+		&mut self,
+		kind: StorageKind,
+		key: &[u8],
+		old_value: Option<&[u8]>,
+		new_value: &[u8],
+	) -> bool {
+		(&mut **self).local_storage_compare_and_set(kind, key, old_value, new_value)
+	}
+
+	fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option<Vec<u8>> {
+		(&mut **self).local_storage_get(kind, key)
+	}
+
+	fn http_request_start(&mut self, method: &str, uri: &str, meta: &[u8]) -> Result<HttpRequestId, ()> {
+		(&mut **self).http_request_start(method, uri, meta)
+	}
+
+	fn http_request_add_header(&mut self, request_id: HttpRequestId, name: &str, value: &str) -> Result<(), ()> {
+		(&mut **self).http_request_add_header(request_id, name, value)
+	}
+
+	fn http_request_write_body(
+		&mut self,
+		request_id: HttpRequestId,
+		chunk: &[u8],
+		deadline: Option<Timestamp>
+	) -> Result<(), HttpError> {
+		(&mut **self).http_request_write_body(request_id, chunk, deadline)
+	}
+
+	fn http_response_wait(&mut self, ids: &[HttpRequestId], deadline: Option<Timestamp>) -> Vec<HttpRequestStatus> {
+		(&mut **self).http_response_wait(ids, deadline)
+	}
+
+	fn http_response_headers(&mut self, request_id: HttpRequestId) -> Vec<(Vec<u8>, Vec<u8>)> {
+		(&mut **self).http_response_headers(request_id)
+	}
+
+	fn http_response_read_body(
+		&mut self,
+		request_id: HttpRequestId,
+		buffer: &mut [u8],
+		deadline: Option<Timestamp>
+	) -> Result<usize, HttpError> {
+		(&mut **self).http_response_read_body(request_id, buffer, deadline)
+	}
+}
+/// An `OffchainExternalities` implementation with limited capabilities.
+pub struct LimitedExternalities<T> {
+	capabilities: Capabilities,
+	externalities: T,
+}
+
+impl<T> LimitedExternalities<T> {
+	/// Create new externalities limited to given `capabilities`.
+	pub fn new(capabilities: Capabilities, externalities: T) -> Self {
+		Self {
+			capabilities,
+			externalities,
+		}
+	}
+
+	/// Check if given capability is allowed.
+	///
+	/// Panics in case it is not.
+	fn check(&self, capability: Capability, name: &'static str) {
+		if !self.capabilities.has(capability) {
+			panic!("Accessing a forbidden API: {}. No: {:?} capability.", name, capability);
+		}
+	}
+}
+
+impl<T: Externalities> Externalities for LimitedExternalities<T> {
+	fn is_validator(&self) -> bool {
+		self.check(Capability::Keystore, "is_validator");
+		self.externalities.is_validator()
+	}
+
+	fn submit_transaction(&mut self, ex: Vec<u8>) -> Result<(), ()> {
+		self.check(Capability::TransactionPool, "submit_transaction");
+		self.externalities.submit_transaction(ex)
+	}
+
+	fn network_state(&self) -> Result<OpaqueNetworkState, ()> {
+		self.check(Capability::NetworkState, "network_state");
+		self.externalities.network_state()
+	}
+
+	fn timestamp(&mut self) -> Timestamp {
+		self.check(Capability::Http, "timestamp");
+		self.externalities.timestamp()
+	}
+
+	fn sleep_until(&mut self, deadline: Timestamp) {
+		self.check(Capability::Http, "sleep_until");
+		self.externalities.sleep_until(deadline)
+	}
+
+	fn random_seed(&mut self) -> [u8; 32] {
+		self.check(Capability::Randomness, "random_seed");
+		self.externalities.random_seed()
+	}
+
+	fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) {
+		self.check(Capability::OffchainWorkerDbWrite, "local_storage_set");
+		self.externalities.local_storage_set(kind, key, value)
+	}
+
+	fn local_storage_compare_and_set(
+		&mut self,
+		kind: StorageKind,
+		key: &[u8],
+		old_value: Option<&[u8]>,
+		new_value: &[u8],
+	) -> bool {
+		self.check(Capability::OffchainWorkerDbWrite, "local_storage_compare_and_set");
+		self.externalities.local_storage_compare_and_set(kind, key, old_value, new_value)
+	}
+
+	fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option<Vec<u8>> {
+		self.check(Capability::OffchainWorkerDbRead, "local_storage_get");
+		self.externalities.local_storage_get(kind, key)
+	}
+
+	fn http_request_start(&mut self, method: &str, uri: &str, meta: &[u8]) -> Result<HttpRequestId, ()> {
+		self.check(Capability::Http, "http_request_start");
+		self.externalities.http_request_start(method, uri, meta)
+	}
+
+	fn http_request_add_header(&mut self, request_id: HttpRequestId, name: &str, value: &str) -> Result<(), ()> {
+		self.check(Capability::Http, "http_request_add_header");
+		self.externalities.http_request_add_header(request_id, name, value)
+	}
+
+	fn http_request_write_body(
+		&mut self,
+		request_id: HttpRequestId,
+		chunk: &[u8],
+		deadline: Option<Timestamp>
+	) -> Result<(), HttpError> {
+		self.check(Capability::Http, "http_request_write_body");
+		self.externalities.http_request_write_body(request_id, chunk, deadline)
+	}
+
+	fn http_response_wait(&mut self, ids: &[HttpRequestId], deadline: Option<Timestamp>) -> Vec<HttpRequestStatus> {
+		self.check(Capability::Http, "http_response_wait");
+		self.externalities.http_response_wait(ids, deadline)
+	}
+
+	fn http_response_headers(&mut self, request_id: HttpRequestId) -> Vec<(Vec<u8>, Vec<u8>)> {
+		self.check(Capability::Http, "http_response_headers");
+		self.externalities.http_response_headers(request_id)
+	}
+
+	fn http_response_read_body(
+		&mut self,
+		request_id: HttpRequestId,
+		buffer: &mut [u8],
+		deadline: Option<Timestamp>
+	) -> Result<usize, HttpError> {
+		self.check(Capability::Http, "http_response_read_body");
+		self.externalities.http_response_read_body(request_id, buffer, deadline)
+	}
+}
+
+#[cfg(feature = "std")]
+externalities::decl_extension! {
+	/// The offchain extension that will be registered at the Substrate externalities.
+	pub struct OffchainExt(Box<dyn Externalities>);
+}
+
+#[cfg(feature = "std")]
+impl OffchainExt {
+	/// Create a new instance of `Self`.
+	pub fn new<O: Externalities + 'static>(offchain: O) -> Self {
+		Self(Box::new(offchain))
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn timestamp_ops() {
+		let t = Timestamp(5);
+		assert_eq!(t.add(Duration::from_millis(10)), Timestamp(15));
+		assert_eq!(t.sub(Duration::from_millis(10)), Timestamp(0));
+		assert_eq!(t.diff(&Timestamp(3)), Duration(2));
+	}
+
+	#[test]
+	fn capabilities() {
+		let none = Capabilities::none();
+		let all = Capabilities::all();
+		let some = Capabilities::from(&[Capability::Keystore, Capability::Randomness][..]);
+
+		assert!(!none.has(Capability::Keystore));
+		assert!(all.has(Capability::Keystore));
+		assert!(some.has(Capability::Keystore));
+		assert!(!none.has(Capability::TransactionPool));
+		assert!(all.has(Capability::TransactionPool));
+		assert!(!some.has(Capability::TransactionPool));
+	}
+}

substrate/primitives/core/src/sandbox.rs

@@ -0,0 +1,220 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Definition of a sandbox environment.
+
+use codec::{Encode, Decode};
+use rstd::vec::Vec;
+
+/// Error error that can be returned from host function.
+#[derive(Encode, Decode)]
+#[derive(crate::RuntimeDebug)]
+pub struct HostError;
+
+/// Representation of a typed wasm value.
+#[derive(Clone, Copy, PartialEq, Encode, Decode)]
+#[derive(crate::RuntimeDebug)]
+pub enum TypedValue {
+	/// Value of 32-bit signed or unsigned integer.
+	#[codec(index = "1")]
+	I32(i32),
+
+	/// Value of 64-bit signed or unsigned integer.
+	#[codec(index = "2")]
+	I64(i64),
+
+	/// Value of 32-bit IEEE 754-2008 floating point number represented as a bit pattern.
+	#[codec(index = "3")]
+	F32(i32),
+
+	/// Value of 64-bit IEEE 754-2008 floating point number represented as a bit pattern.
+	#[codec(index = "4")]
+	F64(i64),
+}
+
+impl TypedValue {
+	/// Returns `Some` if this value of type `I32`.
+	pub fn as_i32(&self) -> Option<i32> {
+		match *self {
+			TypedValue::I32(v) => Some(v),
+			_ => None,
+		}
+	}
+}
+
+#[cfg(feature = "std")]
+impl From<::wasmi::RuntimeValue> for TypedValue {
+	fn from(val: ::wasmi::RuntimeValue) -> TypedValue {
+		use ::wasmi::RuntimeValue;
+		match val {
+			RuntimeValue::I32(v) => TypedValue::I32(v),
+			RuntimeValue::I64(v) => TypedValue::I64(v),
+			RuntimeValue::F32(v) => TypedValue::F32(v.to_bits() as i32),
+			RuntimeValue::F64(v) => TypedValue::F64(v.to_bits() as i64),
+		}
+	}
+}
+
+#[cfg(feature = "std")]
+impl From<TypedValue> for ::wasmi::RuntimeValue {
+	fn from(val: TypedValue) -> ::wasmi::RuntimeValue {
+		use ::wasmi::RuntimeValue;
+		use ::wasmi::nan_preserving_float::{F32, F64};
+		match val {
+			TypedValue::I32(v) => RuntimeValue::I32(v),
+			TypedValue::I64(v) => RuntimeValue::I64(v),
+			TypedValue::F32(v_bits) => RuntimeValue::F32(F32::from_bits(v_bits as u32)),
+			TypedValue::F64(v_bits) => RuntimeValue::F64(F64::from_bits(v_bits as u64)),
+		}
+	}
+}
+
+/// Typed value that can be returned from a function.
+///
+/// Basically a `TypedValue` plus `Unit`, for functions which return nothing.
+#[derive(Clone, Copy, PartialEq, Encode, Decode)]
+#[derive(crate::RuntimeDebug)]
+pub enum ReturnValue {
+	/// For returning nothing.
+	Unit,
+	/// For returning some concrete value.
+	Value(TypedValue),
+}
+
+impl From<TypedValue> for ReturnValue {
+	fn from(v: TypedValue) -> ReturnValue {
+		ReturnValue::Value(v)
+	}
+}
+
+impl ReturnValue {
+	/// Maximum number of bytes `ReturnValue` might occupy when serialized with
+	/// `Codec`.
+	///
+	/// Breakdown:
+	///  1 byte for encoding unit/value variant
+	///  1 byte for encoding value type
+	///  8 bytes for encoding the biggest value types available in wasm: f64, i64.
+	pub const ENCODED_MAX_SIZE: usize = 10;
+}
+
+#[test]
+fn return_value_encoded_max_size() {
+	let encoded = ReturnValue::Value(TypedValue::I64(-1)).encode();
+	assert_eq!(encoded.len(), ReturnValue::ENCODED_MAX_SIZE);
+}
+
+/// Describes an entity to define or import into the environment.
+#[derive(Clone, PartialEq, Eq, Encode, Decode)]
+#[derive(crate::RuntimeDebug)]
+pub enum ExternEntity {
+	/// Function that is specified by an index in a default table of
+	/// a module that creates the sandbox.
+	#[codec(index = "1")]
+	Function(u32),
+
+	/// Linear memory that is specified by some identifier returned by sandbox
+	/// module upon creation new sandboxed memory.
+	#[codec(index = "2")]
+	Memory(u32),
+}
+
+/// An entry in a environment definition table.
+///
+/// Each entry has a two-level name and description of an entity
+/// being defined.
+#[derive(Clone, PartialEq, Eq, Encode, Decode)]
+#[derive(crate::RuntimeDebug)]
+pub struct Entry {
+	/// Module name of which corresponding entity being defined.
+	pub module_name: Vec<u8>,
+	/// Field name in which corresponding entity being defined.
+	pub field_name: Vec<u8>,
+	/// External entity being defined.
+	pub entity: ExternEntity,
+}
+
+/// Definition of runtime that could be used by sandboxed code.
+#[derive(Clone, PartialEq, Eq, Encode, Decode)]
+#[derive(crate::RuntimeDebug)]
+pub struct EnvironmentDefinition {
+	/// Vector of all entries in the environment definition.
+	pub entries: Vec<Entry>,
+}
+
+/// Constant for specifying no limit when creating a sandboxed
+/// memory instance. For FFI purposes.
+pub const MEM_UNLIMITED: u32 = -1i32 as u32;
+
+/// No error happened.
+///
+/// For FFI purposes.
+pub const ERR_OK: u32 = 0;
+
+/// Validation or instantiation error occurred when creating new
+/// sandboxed module instance.
+///
+/// For FFI purposes.
+pub const ERR_MODULE: u32 = -1i32 as u32;
+
+/// Out-of-bounds access attempted with memory or table.
+///
+/// For FFI purposes.
+pub const ERR_OUT_OF_BOUNDS: u32 = -2i32 as u32;
+
+/// Execution error occurred (typically trap).
+///
+/// For FFI purposes.
+pub const ERR_EXECUTION: u32 = -3i32 as u32;
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use std::fmt;
+	use codec::Codec;
+
+	fn roundtrip<S: Codec + PartialEq + fmt::Debug>(s: S) {
+		let encoded = s.encode();
+		assert_eq!(S::decode(&mut &encoded[..]).unwrap(), s);
+	}
+
+	#[test]
+	fn env_def_roundtrip() {
+		roundtrip(EnvironmentDefinition {
+			entries: vec![],
+		});
+
+		roundtrip(EnvironmentDefinition {
+			entries: vec![
+				Entry {
+					module_name: b"kernel"[..].into(),
+					field_name: b"memory"[..].into(),
+					entity: ExternEntity::Memory(1337),
+				},
+			],
+		});
+
+		roundtrip(EnvironmentDefinition {
+			entries: vec![
+				Entry {
+					module_name: b"env"[..].into(),
+					field_name: b"abort"[..].into(),
+					entity: ExternEntity::Function(228),
+				},
+			],
+		});
+	}
+}

substrate/primitives/core/src/sr25519.rs

@@ -0,0 +1,751 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+// tag::description[]
+//! Simple sr25519 (Schnorr-Ristretto) API.
+//!
+//! Note: `CHAIN_CODE_LENGTH` must be equal to `crate::crypto::JUNCTION_ID_LEN`
+//! for this to work.
+// end::description[]
+#[cfg(feature = "full_crypto")]
+use rstd::vec::Vec;
+#[cfg(feature = "full_crypto")]
+use schnorrkel::{signing_context, ExpansionMode, Keypair, SecretKey, MiniSecretKey, PublicKey,
+	derive::{Derivation, ChainCode, CHAIN_CODE_LENGTH}
+};
+#[cfg(feature = "std")]
+use substrate_bip39::mini_secret_from_entropy;
+#[cfg(feature = "std")]
+use bip39::{Mnemonic, Language, MnemonicType};
+#[cfg(feature = "full_crypto")]
+use crate::crypto::{
+	Pair as TraitPair, DeriveJunction, Infallible, SecretStringError
+};
+#[cfg(feature = "std")]
+use crate::crypto::Ss58Codec;
+
+use crate::{crypto::{Public as TraitPublic, UncheckedFrom, CryptoType, Derive}};
+use crate::hash::{H256, H512};
+use codec::{Encode, Decode};
+use rstd::ops::Deref;
+
+#[cfg(feature = "std")]
+use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
+#[cfg(feature = "full_crypto")]
+use schnorrkel::keys::{MINI_SECRET_KEY_LENGTH, SECRET_KEY_LENGTH};
+use runtime_interface::pass_by::PassByInner;
+
+// signing context
+#[cfg(feature = "full_crypto")]
+const SIGNING_CTX: &[u8] = b"substrate";
+
+/// An Schnorrkel/Ristretto x25519 ("sr25519") public key.
+#[cfg_attr(feature = "full_crypto", derive(Hash))]
+#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Encode, Decode, Default, PassByInner)]
+pub struct Public(pub [u8; 32]);
+
+/// An Schnorrkel/Ristretto x25519 ("sr25519") key pair.
+#[cfg(feature = "full_crypto")]
+pub struct Pair(Keypair);
+
+#[cfg(feature = "full_crypto")]
+impl Clone for Pair {
+	fn clone(&self) -> Self {
+		Pair(schnorrkel::Keypair {
+			public: self.0.public,
+			secret: schnorrkel::SecretKey::from_bytes(&self.0.secret.to_bytes()[..])
+				.expect("key is always the correct size; qed")
+		})
+	}
+}
+
+impl AsRef<[u8; 32]> for Public {
+	fn as_ref(&self) -> &[u8; 32] {
+		&self.0
+	}
+}
+
+impl AsRef<[u8]> for Public {
+	fn as_ref(&self) -> &[u8] {
+		&self.0[..]
+	}
+}
+
+impl AsMut<[u8]> for Public {
+	fn as_mut(&mut self) -> &mut [u8] {
+		&mut self.0[..]
+	}
+}
+
+impl Deref for Public {
+    type Target = [u8];
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl From<Public> for [u8; 32] {
+	fn from(x: Public) -> [u8; 32] {
+		x.0
+	}
+}
+
+impl From<Public> for H256 {
+	fn from(x: Public) -> H256 {
+		x.0.into()
+	}
+}
+
+#[cfg(feature = "std")]
+impl std::str::FromStr for Public {
+    type Err = crate::crypto::PublicError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Self::from_ss58check(s)
+    }
+}
+
+impl rstd::convert::TryFrom<&[u8]> for Public {
+	type Error = ();
+
+	fn try_from(data: &[u8]) -> Result<Self, Self::Error> {
+		if data.len() == 32 {
+			let mut inner = [0u8; 32];
+			inner.copy_from_slice(data);
+			Ok(Public(inner))
+		} else {
+			Err(())
+		}
+	}
+}
+
+impl UncheckedFrom<[u8; 32]> for Public {
+	fn unchecked_from(x: [u8; 32]) -> Self {
+		Public::from_raw(x)
+	}
+}
+
+impl UncheckedFrom<H256> for Public {
+	fn unchecked_from(x: H256) -> Self {
+		Public::from_h256(x)
+	}
+}
+
+#[cfg(feature = "std")]
+impl std::fmt::Display for Public {
+	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+		write!(f, "{}", self.to_ss58check())
+	}
+}
+
+impl rstd::fmt::Debug for Public {
+	#[cfg(feature = "std")]
+	fn fmt(&self, f: &mut rstd::fmt::Formatter) -> rstd::fmt::Result {
+		let s = self.to_ss58check();
+		write!(f, "{} ({}...)", crate::hexdisplay::HexDisplay::from(&self.0), &s[0..8])
+	}
+
+	#[cfg(not(feature = "std"))]
+	fn fmt(&self, _: &mut rstd::fmt::Formatter) -> rstd::fmt::Result {
+		Ok(())
+	}
+}
+
+#[cfg(feature = "std")]
+impl Serialize for Public {
+	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> where S: Serializer {
+		serializer.serialize_str(&self.to_ss58check())
+	}
+}
+
+#[cfg(feature = "std")]
+impl<'de> Deserialize<'de> for Public {
+	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> where D: Deserializer<'de> {
+		Public::from_ss58check(&String::deserialize(deserializer)?)
+			.map_err(|e| de::Error::custom(format!("{:?}", e)))
+	}
+}
+
+/// An Schnorrkel/Ristretto x25519 ("sr25519") signature.
+///
+/// Instead of importing it for the local module, alias it to be available as a public type
+#[derive(Encode, Decode, PassByInner)]
+pub struct Signature(pub [u8; 64]);
+
+impl rstd::convert::TryFrom<&[u8]> for Signature {
+	type Error = ();
+
+	fn try_from(data: &[u8]) -> Result<Self, Self::Error> {
+		if data.len() == 64 {
+			let mut inner = [0u8; 64];
+			inner.copy_from_slice(data);
+			Ok(Signature(inner))
+		} else {
+			Err(())
+		}
+	}
+}
+
+impl Clone for Signature {
+	fn clone(&self) -> Self {
+		let mut r = [0u8; 64];
+		r.copy_from_slice(&self.0[..]);
+		Signature(r)
+	}
+}
+
+impl Default for Signature {
+	fn default() -> Self {
+		Signature([0u8; 64])
+	}
+}
+
+impl PartialEq for Signature {
+	fn eq(&self, b: &Self) -> bool {
+		self.0[..] == b.0[..]
+	}
+}
+
+impl Eq for Signature {}
+
+impl From<Signature> for [u8; 64] {
+	fn from(v: Signature) -> [u8; 64] {
+		v.0
+	}
+}
+
+impl From<Signature> for H512 {
+	fn from(v: Signature) -> H512 {
+		H512::from(v.0)
+	}
+}
+
+impl AsRef<[u8; 64]> for Signature {
+	fn as_ref(&self) -> &[u8; 64] {
+		&self.0
+	}
+}
+
+impl AsRef<[u8]> for Signature {
+	fn as_ref(&self) -> &[u8] {
+		&self.0[..]
+	}
+}
+
+impl AsMut<[u8]> for Signature {
+	fn as_mut(&mut self) -> &mut [u8] {
+		&mut self.0[..]
+	}
+}
+
+#[cfg(feature = "full_crypto")]
+impl From<schnorrkel::Signature> for Signature {
+	fn from(s: schnorrkel::Signature) -> Signature {
+		Signature(s.to_bytes())
+	}
+}
+
+impl rstd::fmt::Debug for Signature {
+	#[cfg(feature = "std")]
+	fn fmt(&self, f: &mut rstd::fmt::Formatter) -> rstd::fmt::Result {
+		write!(f, "{}", crate::hexdisplay::HexDisplay::from(&self.0))
+	}
+
+	#[cfg(not(feature = "std"))]
+	fn fmt(&self, _: &mut rstd::fmt::Formatter) -> rstd::fmt::Result {
+		Ok(())
+	}
+}
+
+#[cfg(feature = "full_crypto")]
+impl rstd::hash::Hash for Signature {
+	fn hash<H: rstd::hash::Hasher>(&self, state: &mut H) {
+		rstd::hash::Hash::hash(&self.0[..], state);
+	}
+}
+
+/// A localized signature also contains sender information.
+/// NOTE: Encode and Decode traits are supported in ed25519 but not possible for now here.
+#[cfg(feature = "std")]
+#[derive(PartialEq, Eq, Clone, Debug)]
+pub struct LocalizedSignature {
+	/// The signer of the signature.
+	pub signer: Public,
+	/// The signature itself.
+	pub signature: Signature,
+}
+
+impl Signature {
+	/// A new instance from the given 64-byte `data`.
+	///
+	/// NOTE: No checking goes on to ensure this is a real signature. Only use
+	/// it if you are certain that the array actually is a signature, or if you
+	/// immediately verify the signature.  All functions that verify signatures
+	/// will fail if the `Signature` is not actually a valid signature.
+	pub fn from_raw(data: [u8; 64]) -> Signature {
+		Signature(data)
+	}
+
+	/// A new instance from the given slice that should be 64 bytes long.
+	///
+	/// NOTE: No checking goes on to ensure this is a real signature. Only use it if
+	/// you are certain that the array actually is a signature. GIGO!
+	pub fn from_slice(data: &[u8]) -> Self {
+		let mut r = [0u8; 64];
+		r.copy_from_slice(data);
+		Signature(r)
+	}
+
+	/// A new instance from an H512.
+	///
+	/// NOTE: No checking goes on to ensure this is a real signature. Only use it if
+	/// you are certain that the array actually is a signature. GIGO!
+	pub fn from_h512(v: H512) -> Signature {
+		Signature(v.into())
+	}
+}
+
+impl Derive for Public {
+	/// Derive a child key from a series of given junctions.
+	///
+	/// `None` if there are any hard junctions in there.
+	#[cfg(feature = "std")]
+	fn derive<Iter: Iterator<Item=DeriveJunction>>(&self, path: Iter) -> Option<Public> {
+		let mut acc = PublicKey::from_bytes(self.as_ref()).ok()?;
+		for j in path {
+			match j {
+				DeriveJunction::Soft(cc) => acc = acc.derived_key_simple(ChainCode(cc), &[]).0,
+				DeriveJunction::Hard(_cc) => return None,
+			}
+		}
+		Some(Self(acc.to_bytes()))
+	}
+}
+
+impl Public {
+	/// A new instance from the given 32-byte `data`.
+	///
+	/// NOTE: No checking goes on to ensure this is a real public key. Only use it if
+	/// you are certain that the array actually is a pubkey. GIGO!
+	pub fn from_raw(data: [u8; 32]) -> Self {
+		Public(data)
+	}
+
+	/// A new instance from an H256.
+	///
+	/// NOTE: No checking goes on to ensure this is a real public key. Only use it if
+	/// you are certain that the array actually is a pubkey. GIGO!
+	pub fn from_h256(x: H256) -> Self {
+		Public(x.into())
+	}
+
+	/// Return a slice filled with raw data.
+	pub fn as_array_ref(&self) -> &[u8; 32] {
+		self.as_ref()
+	}
+}
+
+impl TraitPublic for Public {
+	/// A new instance from the given slice that should be 32 bytes long.
+	///
+	/// NOTE: No checking goes on to ensure this is a real public key. Only use it if
+	/// you are certain that the array actually is a pubkey. GIGO!
+	fn from_slice(data: &[u8]) -> Self {
+		let mut r = [0u8; 32];
+		r.copy_from_slice(data);
+		Public(r)
+	}
+}
+
+#[cfg(feature = "std")]
+impl From<MiniSecretKey> for Pair {
+	fn from(sec: MiniSecretKey) -> Pair {
+		Pair(sec.expand_to_keypair(ExpansionMode::Ed25519))
+	}
+}
+
+#[cfg(feature = "std")]
+impl From<SecretKey> for Pair {
+	fn from(sec: SecretKey) -> Pair {
+		Pair(Keypair::from(sec))
+	}
+}
+
+#[cfg(feature = "full_crypto")]
+impl From<schnorrkel::Keypair> for Pair {
+	fn from(p: schnorrkel::Keypair) -> Pair {
+		Pair(p)
+	}
+}
+
+#[cfg(feature = "full_crypto")]
+impl From<Pair> for schnorrkel::Keypair {
+	fn from(p: Pair) -> schnorrkel::Keypair {
+		p.0
+	}
+}
+
+#[cfg(feature = "full_crypto")]
+impl AsRef<schnorrkel::Keypair> for Pair {
+	fn as_ref(&self) -> &schnorrkel::Keypair {
+		&self.0
+	}
+}
+
+/// Derive a single hard junction.
+#[cfg(feature = "full_crypto")]
+fn derive_hard_junction(secret: &SecretKey, cc: &[u8; CHAIN_CODE_LENGTH]) -> MiniSecretKey {
+	secret.hard_derive_mini_secret_key(Some(ChainCode(cc.clone())), b"").0
+}
+
+/// The raw secret seed, which can be used to recreate the `Pair`.
+#[cfg(feature = "full_crypto")]
+type Seed = [u8; MINI_SECRET_KEY_LENGTH];
+
+#[cfg(feature = "full_crypto")]
+impl TraitPair for Pair {
+	type Public = Public;
+	type Seed = Seed;
+	type Signature = Signature;
+	type DeriveError = Infallible;
+
+	/// Make a new key pair from raw secret seed material.
+	///
+	/// This is generated using schnorrkel's Mini-Secret-Keys.
+	///
+	/// A MiniSecretKey is literally what Ed25519 calls a SecretKey, which is just 32 random bytes.
+	fn from_seed(seed: &Seed) -> Pair {
+		Self::from_seed_slice(&seed[..])
+			.expect("32 bytes can always build a key; qed")
+	}
+
+	/// Get the public key.
+	fn public(&self) -> Public {
+		let mut pk = [0u8; 32];
+		pk.copy_from_slice(&self.0.public.to_bytes());
+		Public(pk)
+	}
+
+	/// Make a new key pair from secret seed material. The slice must be 32 bytes long or it
+	/// will return `None`.
+	///
+	/// You should never need to use this; generate(), generate_with_phrase(), from_phrase()
+	fn from_seed_slice(seed: &[u8]) -> Result<Pair, SecretStringError> {
+		match seed.len() {
+			MINI_SECRET_KEY_LENGTH => {
+				Ok(Pair(
+					MiniSecretKey::from_bytes(seed)
+						.map_err(|_| SecretStringError::InvalidSeed)?
+						.expand_to_keypair(ExpansionMode::Ed25519)
+				))
+			}
+			SECRET_KEY_LENGTH => {
+				Ok(Pair(
+					SecretKey::from_bytes(seed)
+						.map_err(|_| SecretStringError::InvalidSeed)?
+						.to_keypair()
+				))
+			}
+			_ => Err(SecretStringError::InvalidSeedLength)
+		}
+	}
+	#[cfg(feature = "std")]
+	fn generate_with_phrase(password: Option<&str>) -> (Pair, String, Seed) {
+		let mnemonic = Mnemonic::new(MnemonicType::Words12, Language::English);
+		let phrase = mnemonic.phrase();
+		let (pair, seed) = Self::from_phrase(phrase, password)
+			.expect("All phrases generated by Mnemonic are valid; qed");
+		(
+			pair,
+			phrase.to_owned(),
+			seed,
+		)
+	}
+	#[cfg(feature = "std")]
+	fn from_phrase(phrase: &str, password: Option<&str>) -> Result<(Pair, Seed), SecretStringError> {
+		Mnemonic::from_phrase(phrase, Language::English)
+			.map_err(|_| SecretStringError::InvalidPhrase)
+			.map(|m| Self::from_entropy(m.entropy(), password))
+	}
+
+	fn derive<Iter: Iterator<Item=DeriveJunction>>(&self,
+		path: Iter,
+		seed: Option<Seed>,
+	) -> Result<(Pair, Option<Seed>), Self::DeriveError> {
+		let seed = if let Some(s) = seed {
+			if let Ok(msk) = MiniSecretKey::from_bytes(&s) {
+				if msk.expand(ExpansionMode::Ed25519) == self.0.secret {
+					Some(msk)
+				} else { None }
+			} else { None }
+		} else { None };
+		let init = self.0.secret.clone();
+		let (result, seed) = path.fold((init, seed), |(acc, acc_seed), j| match (j, acc_seed) {
+			(DeriveJunction::Soft(cc), _) =>
+				(acc.derived_key_simple(ChainCode(cc), &[]).0, None),
+			(DeriveJunction::Hard(cc), maybe_seed) => {
+				let seed = derive_hard_junction(&acc, &cc);
+				(seed.expand(ExpansionMode::Ed25519), maybe_seed.map(|_| seed))
+			}
+		});
+		Ok((Self(result.into()), seed.map(|s| MiniSecretKey::to_bytes(&s))))
+	}
+
+	fn sign(&self, message: &[u8]) -> Signature {
+		let context = signing_context(SIGNING_CTX);
+		self.0.sign(context.bytes(message)).into()
+	}
+
+	/// Verify a signature on a message. Returns true if the signature is good.
+	fn verify<M: AsRef<[u8]>>(sig: &Self::Signature, message: M, pubkey: &Self::Public) -> bool {
+		Self::verify_weak(&sig.0[..], message, pubkey)
+	}
+
+	/// Verify a signature on a message. Returns true if the signature is good.
+	fn verify_weak<P: AsRef<[u8]>, M: AsRef<[u8]>>(sig: &[u8], message: M, pubkey: P) -> bool {
+		// Match both schnorrkel 0.1.1 and 0.8.0+ signatures, supporting both wallets
+		// that have not been upgraded and those that have. To swap to 0.8.0 only,
+		// create `schnorrkel::Signature` and pass that into `verify_simple`
+		match PublicKey::from_bytes(pubkey.as_ref()) {
+			Ok(pk) => pk.verify_simple_preaudit_deprecated(
+				SIGNING_CTX, message.as_ref(), &sig,
+			).is_ok(),
+			Err(_) => false,
+		}
+	}
+
+	/// Return a vec filled with raw data.
+	fn to_raw_vec(&self) -> Vec<u8> {
+		self.0.secret.to_bytes().to_vec()
+	}
+}
+
+#[cfg(feature = "std")]
+impl Pair {
+	/// Make a new key pair from binary data derived from a valid seed phrase.
+	///
+	/// This uses a key derivation function to convert the entropy into a seed, then returns
+	/// the pair generated from it.
+	pub fn from_entropy(entropy: &[u8], password: Option<&str>) -> (Pair, Seed) {
+		let mini_key: MiniSecretKey = mini_secret_from_entropy(entropy, password.unwrap_or(""))
+			.expect("32 bytes can always build a key; qed");
+
+		let kp = mini_key.expand_to_keypair(ExpansionMode::Ed25519);
+		(Pair(kp), mini_key.to_bytes())
+	}
+}
+
+impl CryptoType for Public {
+	#[cfg(feature = "full_crypto")]
+	type Pair = Pair;
+}
+
+impl CryptoType for Signature {
+	#[cfg(feature = "full_crypto")]
+	type Pair = Pair;
+}
+
+#[cfg(feature = "full_crypto")]
+impl CryptoType for Pair {
+	type Pair = Pair;
+}
+
+#[cfg(test)]
+mod compatibility_test {
+	use super::*;
+	use crate::crypto::{DEV_PHRASE};
+	use hex_literal::hex;
+
+	// NOTE: tests to ensure addresses that are created with the `0.1.x` version (pre-audit) are
+	// still functional.
+
+	#[test]
+	fn derive_soft_known_pair_should_work() {
+		let pair = Pair::from_string(&format!("{}/Alice", DEV_PHRASE), None).unwrap();
+		// known address of DEV_PHRASE with 1.1
+		let known = hex!("d6c71059dbbe9ad2b0ed3f289738b800836eb425544ce694825285b958ca755e");
+		assert_eq!(pair.public().to_raw_vec(), known);
+	}
+
+	#[test]
+	fn derive_hard_known_pair_should_work() {
+		let pair = Pair::from_string(&format!("{}//Alice", DEV_PHRASE), None).unwrap();
+		// known address of DEV_PHRASE with 1.1
+		let known = hex!("d43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d");
+		assert_eq!(pair.public().to_raw_vec(), known);
+	}
+
+	#[test]
+	fn verify_known_message_should_work() {
+		let public = Public::from_raw(hex!("b4bfa1f7a5166695eb75299fd1c4c03ea212871c342f2c5dfea0902b2c246918"));
+		// signature generated by the 1.1 version with the same ^^ public key.
+		let signature = Signature::from_raw(hex!(
+			"5a9755f069939f45d96aaf125cf5ce7ba1db998686f87f2fb3cbdea922078741a73891ba265f70c31436e18a9acd14d189d73c12317ab6c313285cd938453202"
+		));
+		let message = b"Verifying that I am the owner of 5G9hQLdsKQswNPgB499DeA5PkFBbgkLPJWkkS6FAM6xGQ8xD. Hash: 221455a3\n";
+		assert!(Pair::verify(&signature, &message[..], &public));
+	}
+}
+
+#[cfg(test)]
+mod test {
+	use super::*;
+	use crate::crypto::{Ss58Codec, DEV_PHRASE, DEV_ADDRESS};
+	use hex_literal::hex;
+
+	#[test]
+	fn default_phrase_should_be_used() {
+		assert_eq!(
+			Pair::from_string("//Alice///password", None).unwrap().public(),
+			Pair::from_string(&format!("{}//Alice", DEV_PHRASE), Some("password")).unwrap().public(),
+		);
+		assert_eq!(
+			Pair::from_string(&format!("{}/Alice", DEV_PHRASE), None).as_ref().map(Pair::public),
+			Pair::from_string("/Alice", None).as_ref().map(Pair::public)
+		);
+	}
+
+	#[test]
+	fn default_address_should_be_used() {
+		assert_eq!(
+			Public::from_string(&format!("{}/Alice", DEV_ADDRESS)),
+			Public::from_string("/Alice")
+		);
+	}
+
+	#[test]
+	fn default_phrase_should_correspond_to_default_address() {
+		assert_eq!(
+			Pair::from_string(&format!("{}/Alice", DEV_PHRASE), None).unwrap().public(),
+			Public::from_string(&format!("{}/Alice", DEV_ADDRESS)).unwrap(),
+		);
+		assert_eq!(
+			Pair::from_string("/Alice", None).unwrap().public(),
+			Public::from_string("/Alice").unwrap()
+		);
+	}
+
+	#[test]
+	fn derive_soft_should_work() {
+		let pair = Pair::from_seed(&hex!(
+			"9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60"
+		));
+		let derive_1 = pair.derive(Some(DeriveJunction::soft(1)).into_iter(), None).unwrap().0;
+		let derive_1b = pair.derive(Some(DeriveJunction::soft(1)).into_iter(), None).unwrap().0;
+		let derive_2 = pair.derive(Some(DeriveJunction::soft(2)).into_iter(), None).unwrap().0;
+		assert_eq!(derive_1.public(), derive_1b.public());
+		assert_ne!(derive_1.public(), derive_2.public());
+	}
+
+	#[test]
+	fn derive_hard_should_work() {
+		let pair = Pair::from_seed(&hex!(
+			"9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60"
+		));
+		let derive_1 = pair.derive(Some(DeriveJunction::hard(1)).into_iter(), None).unwrap().0;
+		let derive_1b = pair.derive(Some(DeriveJunction::hard(1)).into_iter(), None).unwrap().0;
+		let derive_2 = pair.derive(Some(DeriveJunction::hard(2)).into_iter(), None).unwrap().0;
+		assert_eq!(derive_1.public(), derive_1b.public());
+		assert_ne!(derive_1.public(), derive_2.public());
+	}
+
+	#[test]
+	fn derive_soft_public_should_work() {
+		let pair = Pair::from_seed(&hex!(
+			"9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60"
+		));
+		let path = Some(DeriveJunction::soft(1));
+		let pair_1 = pair.derive(path.clone().into_iter(), None).unwrap().0;
+		let public_1 = pair.public().derive(path.into_iter()).unwrap();
+		assert_eq!(pair_1.public(), public_1);
+	}
+
+	#[test]
+	fn derive_hard_public_should_fail() {
+		let pair = Pair::from_seed(&hex!(
+			"9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60"
+		));
+		let path = Some(DeriveJunction::hard(1));
+		assert!(pair.public().derive(path.into_iter()).is_none());
+	}
+
+	#[test]
+	fn sr_test_vector_should_work() {
+		let pair = Pair::from_seed(&hex!(
+			"9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60"
+		));
+		let public = pair.public();
+		assert_eq!(
+			public,
+			Public::from_raw(hex!(
+				"44a996beb1eef7bdcab976ab6d2ca26104834164ecf28fb375600576fcc6eb0f"
+			))
+		);
+		let message = b"";
+		let signature = pair.sign(message);
+		assert!(Pair::verify(&signature, &message[..], &public));
+	}
+
+	#[test]
+	fn generated_pair_should_work() {
+		let (pair, _) = Pair::generate();
+		let public = pair.public();
+		let message = b"Something important";
+		let signature = pair.sign(&message[..]);
+		assert!(Pair::verify(&signature, &message[..], &public));
+	}
+
+	#[test]
+	fn seeded_pair_should_work() {
+		let pair = Pair::from_seed(b"12345678901234567890123456789012");
+		let public = pair.public();
+		assert_eq!(
+			public,
+			Public::from_raw(hex!(
+				"741c08a06f41c596608f6774259bd9043304adfa5d3eea62760bd9be97634d63"
+			))
+		);
+		let message = hex!("2f8c6129d816cf51c374bc7f08c3e63ed156cf78aefb4a6550d97b87997977ee00000000000000000200d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a4500000000000000");
+		let signature = pair.sign(&message[..]);
+		assert!(Pair::verify(&signature, &message[..], &public));
+	}
+
+	#[test]
+	fn ss58check_roundtrip_works() {
+		let (pair, _) = Pair::generate();
+		let public = pair.public();
+		let s = public.to_ss58check();
+		println!("Correct: {}", s);
+		let cmp = Public::from_ss58check(&s).unwrap();
+		assert_eq!(cmp, public);
+	}
+
+	#[test]
+	fn verify_from_wasm_works() {
+		// The values in this test case are compared to the output of `node-test.js` in schnorrkel-js.
+		//
+		// This is to make sure that the wasm library is compatible.
+		let pk = Pair::from_seed(
+			&hex!("0000000000000000000000000000000000000000000000000000000000000000")
+		);
+		let public = pk.public();
+		let js_signature = Signature::from_raw(hex!(
+			"28a854d54903e056f89581c691c1f7d2ff39f8f896c9e9c22475e60902cc2b3547199e0e91fa32902028f2ca2355e8cdd16cfe19ba5e8b658c94aa80f3b81a00"
+		));
+		assert!(Pair::verify(&js_signature, b"SUBSTRATE", &public));
+	}
+}

substrate/primitives/core/src/testing.rs

@@ -0,0 +1,273 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Types that should only be used for testing!
+
+#[cfg(feature = "std")]
+use crate::{ed25519, sr25519, crypto::{Public, Pair}};
+use crate::crypto::KeyTypeId;
+
+/// Key type for generic Ed25519 key.
+pub const ED25519: KeyTypeId = KeyTypeId(*b"ed25");
+/// Key type for generic Sr 25519 key.
+pub const SR25519: KeyTypeId = KeyTypeId(*b"sr25");
+
+/// A keystore implementation usable in tests.
+#[cfg(feature = "std")]
+#[derive(Default)]
+pub struct KeyStore {
+	/// `KeyTypeId` maps to public keys and public keys map to private keys.
+	keys: std::collections::HashMap<KeyTypeId, std::collections::HashMap<Vec<u8>, String>>,
+}
+
+#[cfg(feature = "std")]
+impl KeyStore {
+	/// Creates a new instance of `Self`.
+	pub fn new() -> crate::traits::BareCryptoStorePtr {
+		std::sync::Arc::new(parking_lot::RwLock::new(Self::default()))
+	}
+}
+
+#[cfg(feature = "std")]
+impl crate::traits::BareCryptoStore for KeyStore {
+	fn sr25519_public_keys(&self, id: KeyTypeId) -> Vec<sr25519::Public> {
+		self.keys.get(&id)
+			.map(|keys|
+				keys.values()
+					.map(|s| sr25519::Pair::from_string(s, None).expect("`sr25519` seed slice is valid"))
+					.map(|p| p.public())
+					.collect()
+			)
+			.unwrap_or_default()
+	}
+
+	fn sr25519_generate_new(
+		&mut self,
+		id: KeyTypeId,
+		seed: Option<&str>,
+	) -> Result<sr25519::Public, String> {
+		match seed {
+			Some(seed) => {
+				let pair = sr25519::Pair::from_string(seed, None).expect("Generates an `sr25519` pair.");
+				self.keys.entry(id).or_default().insert(pair.public().to_raw_vec(), seed.into());
+				Ok(pair.public())
+			},
+			None => {
+				let (pair, phrase, _) = sr25519::Pair::generate_with_phrase(None);
+				self.keys.entry(id).or_default().insert(pair.public().to_raw_vec(), phrase);
+				Ok(pair.public())
+			}
+		}
+	}
+
+	fn sr25519_key_pair(&self, id: KeyTypeId, pub_key: &sr25519::Public) -> Option<sr25519::Pair> {
+		self.keys.get(&id)
+			.and_then(|inner|
+				inner.get(pub_key.as_slice())
+					.map(|s| sr25519::Pair::from_string(s, None).expect("`sr25519` seed slice is valid"))
+			)
+	}
+
+	fn ed25519_public_keys(&self, id: KeyTypeId) -> Vec<ed25519::Public> {
+		self.keys.get(&id)
+			.map(|keys|
+				keys.values()
+					.map(|s| ed25519::Pair::from_string(s, None).expect("`ed25519` seed slice is valid"))
+					.map(|p| p.public())
+					.collect()
+			)
+			.unwrap_or_default()
+	}
+
+	fn ed25519_generate_new(
+		&mut self,
+		id: KeyTypeId,
+		seed: Option<&str>,
+	) -> Result<ed25519::Public, String> {
+		match seed {
+			Some(seed) => {
+				let pair = ed25519::Pair::from_string(seed, None).expect("Generates an `ed25519` pair.");
+				self.keys.entry(id).or_default().insert(pair.public().to_raw_vec(), seed.into());
+				Ok(pair.public())
+			},
+			None => {
+				let (pair, phrase, _) = ed25519::Pair::generate_with_phrase(None);
+				self.keys.entry(id).or_default().insert(pair.public().to_raw_vec(), phrase);
+				Ok(pair.public())
+			}
+		}
+	}
+
+	fn ed25519_key_pair(&self, id: KeyTypeId, pub_key: &ed25519::Public) -> Option<ed25519::Pair> {
+		self.keys.get(&id)
+			.and_then(|inner|
+				inner.get(pub_key.as_slice())
+					.map(|s| ed25519::Pair::from_string(s, None).expect("`ed25519` seed slice is valid"))
+			)
+	}
+
+	fn insert_unknown(&mut self, id: KeyTypeId, suri: &str, public: &[u8]) -> Result<(), ()> {
+		self.keys.entry(id).or_default().insert(public.to_owned(), suri.to_string());
+		Ok(())
+	}
+
+	fn password(&self) -> Option<&str> {
+		None
+	}
+}
+
+/// Macro for exporting functions from wasm in with the expected signature for using it with the
+/// wasm executor. This is useful for tests where you need to call a function in wasm.
+///
+/// The input parameters are expected to be SCALE encoded and will be automatically decoded for you.
+/// The output value is also SCALE encoded when returned back to the host.
+///
+/// The functions are feature-gated with `#[cfg(not(feature = "std"))]`, so they are only available
+/// from within wasm.
+///
+/// # Example
+///
+/// ```
+/// # use substrate_primitives::wasm_export_functions;
+///
+/// wasm_export_functions! {
+///     fn test_in_wasm(value: bool, another_value: Vec<u8>) -> bool {
+///         value && another_value.is_empty()
+///     }
+///
+///     fn without_return_value() {
+///         // do something
+///     }
+/// }
+/// ```
+#[macro_export]
+macro_rules! wasm_export_functions {
+	(
+		$(
+			fn $name:ident (
+				$( $arg_name:ident: $arg_ty:ty ),* $(,)?
+			) $( -> $ret_ty:ty )? { $( $fn_impl:tt )* }
+		)*
+	) => {
+		$(
+			$crate::wasm_export_functions! {
+				@IMPL
+				fn $name (
+					$( $arg_name: $arg_ty ),*
+				) $( -> $ret_ty )? { $( $fn_impl )* }
+			}
+		)*
+	};
+	(@IMPL
+		fn $name:ident (
+				$( $arg_name:ident: $arg_ty:ty ),*
+		) { $( $fn_impl:tt )* }
+	) => {
+		#[no_mangle]
+		#[allow(unreachable_code)]
+		#[cfg(not(feature = "std"))]
+		pub fn $name(input_data: *mut u8, input_len: usize) -> u64 {
+			let input: &[u8] = if input_len == 0 {
+				&[0u8; 0]
+			} else {
+				unsafe {
+					$crate::rstd::slice::from_raw_parts(input_data, input_len)
+				}
+			};
+
+			{
+				let ($( $arg_name ),*) : ($( $arg_ty ),*) = $crate::Decode::decode(
+					&mut &input[..],
+				).expect("Input data is correctly encoded");
+
+				$( $fn_impl )*
+			}
+
+			$crate::to_substrate_wasm_fn_return_value(&())
+		}
+	};
+	(@IMPL
+		fn $name:ident (
+				$( $arg_name:ident: $arg_ty:ty ),*
+		) $( -> $ret_ty:ty )? { $( $fn_impl:tt )* }
+	) => {
+		#[no_mangle]
+		#[allow(unreachable_code)]
+		#[cfg(not(feature = "std"))]
+		pub fn $name(input_data: *mut u8, input_len: usize) -> u64 {
+			let input: &[u8] = if input_len == 0 {
+				&[0u8; 0]
+			} else {
+				unsafe {
+					$crate::rstd::slice::from_raw_parts(input_data, input_len)
+				}
+			};
+
+			let output $( : $ret_ty )? = {
+				let ($( $arg_name ),*) : ($( $arg_ty ),*) = $crate::Decode::decode(
+					&mut &input[..],
+				).expect("Input data is correctly encoded");
+
+				$( $fn_impl )*
+			};
+
+			$crate::to_substrate_wasm_fn_return_value(&output)
+		}
+	};
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use crate::sr25519;
+	use crate::testing::{ED25519, SR25519};
+
+	#[test]
+	fn store_key_and_extract() {
+		let store = KeyStore::new();
+
+		let public = store.write()
+			.ed25519_generate_new(ED25519, None)
+			.expect("Genrates key");
+
+		let store_key_pair = store.read()
+			.ed25519_key_pair(ED25519, &public)
+			.expect("Key should exists in store");
+
+		assert_eq!(public, store_key_pair.public());
+	}
+
+	#[test]
+	fn store_unknown_and_extract_it() {
+		let store = KeyStore::new();
+
+		let secret_uri = "//Alice";
+		let key_pair = sr25519::Pair::from_string(secret_uri, None).expect("Generates key pair");
+
+		store.write().insert_unknown(
+			SR25519,
+			secret_uri,
+			key_pair.public().as_ref(),
+		).expect("Inserts unknown key");
+
+		let store_key_pair = store.read().sr25519_key_pair(
+			SR25519,
+			&key_pair.public(),
+		).expect("Gets key pair from keystore");
+
+		assert_eq!(key_pair.public(), store_key_pair.public());
+	}
+}

substrate/primitives/core/src/tests.rs

@@ -0,0 +1,17 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Tests.

substrate/primitives/core/src/traits.rs

@@ -0,0 +1,97 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Shareable Substrate traits.
+
+use crate::{crypto::KeyTypeId, ed25519, sr25519};
+
+use std::{fmt::{Debug, Display}, panic::UnwindSafe};
+
+pub use externalities::{Externalities, ExternalitiesExt};
+
+/// Something that generates, stores and provides access to keys.
+pub trait BareCryptoStore: Send + Sync {
+	/// Returns all sr25519 public keys for the given key type.
+	fn sr25519_public_keys(&self, id: KeyTypeId) -> Vec<sr25519::Public>;
+	/// Generate a new sr25519 key pair for the given key type and an optional seed.
+	///
+	/// If the given seed is `Some(_)`, the key pair will only be stored in memory.
+	///
+	/// Returns the public key of the generated key pair.
+	fn sr25519_generate_new(
+		&mut self,
+		id: KeyTypeId,
+		seed: Option<&str>,
+	) -> Result<sr25519::Public, String>;
+	/// Returns the sr25519 key pair for the given key type and public key combination.
+	fn sr25519_key_pair(&self, id: KeyTypeId, pub_key: &sr25519::Public) -> Option<sr25519::Pair>;
+
+	/// Returns all ed25519 public keys for the given key type.
+	fn ed25519_public_keys(&self, id: KeyTypeId) -> Vec<ed25519::Public>;
+	/// Generate a new ed25519 key pair for the given key type and an optional seed.
+	///
+	/// If the given seed is `Some(_)`, the key pair will only be stored in memory.
+	///
+	/// Returns the public key of the generated key pair.
+	fn ed25519_generate_new(
+		&mut self,
+		id: KeyTypeId,
+		seed: Option<&str>,
+	) -> Result<ed25519::Public, String>;
+
+	/// Returns the ed25519 key pair for the given key type and public key combination.
+	fn ed25519_key_pair(&self, id: KeyTypeId, pub_key: &ed25519::Public) -> Option<ed25519::Pair>;
+
+	/// Insert a new key. This doesn't require any known of the crypto; but a public key must be
+	/// manually provided.
+	///
+	/// Places it into the file system store.
+	///
+	/// `Err` if there's some sort of weird filesystem error, but should generally be `Ok`.
+	fn insert_unknown(&mut self, _key_type: KeyTypeId, _suri: &str, _public: &[u8]) -> Result<(), ()>;
+
+	/// Get the password for this store.
+	fn password(&self) -> Option<&str>;
+}
+
+/// A pointer to the key store.
+pub type BareCryptoStorePtr = std::sync::Arc<parking_lot::RwLock<dyn BareCryptoStore>>;
+
+externalities::decl_extension! {
+	/// The keystore extension to register/retrieve from the externalities.
+	pub struct KeystoreExt(BareCryptoStorePtr);
+}
+
+/// Code execution engine.
+pub trait CodeExecutor: Sized + Send + Sync {
+	/// Externalities error type.
+	type Error: Display + Debug + Send + 'static;
+
+	/// Call a given method in the runtime. Returns a tuple of the result (either the output data
+	/// or an execution error) together with a `bool`, which is true if native execution was used.
+	fn call<
+		E: Externalities,
+		R: codec::Codec + PartialEq,
+		NC: FnOnce() -> Result<R, String> + UnwindSafe,
+	>(
+		&self,
+		ext: &mut E,
+		method: &str,
+		data: &[u8],
+		use_native: bool,
+		native_call: Option<NC>,
+	) -> (Result<crate::NativeOrEncoded<R>, Self::Error>, bool);
+}

substrate/primitives/core/src/u32_trait.rs

@@ -0,0 +1,243 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! An u32 trait with "values" as impl'd types.
+
+/// A u32 value, wrapped in a trait because we don't yet have const generics.
+pub trait Value {
+	/// The actual value represented by the impl'ing type.
+	const VALUE: u32;
+}
+
+/// Type representing the value 0 for the `Value` trait.
+pub struct _0; impl Value for _0 { const VALUE: u32 = 0; }
+/// Type representing the value 1 for the `Value` trait.
+pub struct _1; impl Value for _1 { const VALUE: u32 = 1; }
+/// Type representing the value 2 for the `Value` trait.
+pub struct _2; impl Value for _2 { const VALUE: u32 = 2; }
+/// Type representing the value 3 for the `Value` trait.
+pub struct _3; impl Value for _3 { const VALUE: u32 = 3; }
+/// Type representing the value 4 for the `Value` trait.
+pub struct _4; impl Value for _4 { const VALUE: u32 = 4; }
+/// Type representing the value 5 for the `Value` trait.
+pub struct _5; impl Value for _5 { const VALUE: u32 = 5; }
+/// Type representing the value 6 for the `Value` trait.
+pub struct _6; impl Value for _6 { const VALUE: u32 = 6; }
+/// Type representing the value 7 for the `Value` trait.
+pub struct _7; impl Value for _7 { const VALUE: u32 = 7; }
+/// Type representing the value 8 for the `Value` trait.
+pub struct _8; impl Value for _8 { const VALUE: u32 = 8; }
+/// Type representing the value 9 for the `Value` trait.
+pub struct _9; impl Value for _9 { const VALUE: u32 = 9; }
+/// Type representing the value 10 for the `Value` trait.
+pub struct _10; impl Value for _10 { const VALUE: u32 = 10; }
+/// Type representing the value 11 for the `Value` trait.
+pub struct _11; impl Value for _11 { const VALUE: u32 = 11; }
+/// Type representing the value 12 for the `Value` trait.
+pub struct _12; impl Value for _12 { const VALUE: u32 = 12; }
+/// Type representing the value 13 for the `Value` trait.
+pub struct _13; impl Value for _13 { const VALUE: u32 = 13; }
+/// Type representing the value 14 for the `Value` trait.
+pub struct _14; impl Value for _14 { const VALUE: u32 = 14; }
+/// Type representing the value 15 for the `Value` trait.
+pub struct _15; impl Value for _15 { const VALUE: u32 = 15; }
+/// Type representing the value 16 for the `Value` trait.
+pub struct _16; impl Value for _16 { const VALUE: u32 = 16; }
+/// Type representing the value 17 for the `Value` trait.
+pub struct _17; impl Value for _17 { const VALUE: u32 = 17; }
+/// Type representing the value 18 for the `Value` trait.
+pub struct _18; impl Value for _18 { const VALUE: u32 = 18; }
+/// Type representing the value 19 for the `Value` trait.
+pub struct _19; impl Value for _19 { const VALUE: u32 = 19; }
+/// Type representing the value 20 for the `Value` trait.
+pub struct _20; impl Value for _20 { const VALUE: u32 = 20; }
+/// Type representing the value 21 for the `Value` trait.
+pub struct _21; impl Value for _21 { const VALUE: u32 = 21; }
+/// Type representing the value 22 for the `Value` trait.
+pub struct _22; impl Value for _22 { const VALUE: u32 = 22; }
+/// Type representing the value 23 for the `Value` trait.
+pub struct _23; impl Value for _23 { const VALUE: u32 = 23; }
+/// Type representing the value 24 for the `Value` trait.
+pub struct _24; impl Value for _24 { const VALUE: u32 = 24; }
+/// Type representing the value 25 for the `Value` trait.
+pub struct _25; impl Value for _25 { const VALUE: u32 = 25; }
+/// Type representing the value 26 for the `Value` trait.
+pub struct _26; impl Value for _26 { const VALUE: u32 = 26; }
+/// Type representing the value 27 for the `Value` trait.
+pub struct _27; impl Value for _27 { const VALUE: u32 = 27; }
+/// Type representing the value 28 for the `Value` trait.
+pub struct _28; impl Value for _28 { const VALUE: u32 = 28; }
+/// Type representing the value 29 for the `Value` trait.
+pub struct _29; impl Value for _29 { const VALUE: u32 = 29; }
+/// Type representing the value 30 for the `Value` trait.
+pub struct _30; impl Value for _30 { const VALUE: u32 = 30; }
+/// Type representing the value 31 for the `Value` trait.
+pub struct _31; impl Value for _31 { const VALUE: u32 = 31; }
+/// Type representing the value 32 for the `Value` trait.
+pub struct _32; impl Value for _32 { const VALUE: u32 = 32; }
+/// Type representing the value 33 for the `Value` trait.
+pub struct _33; impl Value for _33 { const VALUE: u32 = 33; }
+/// Type representing the value 34 for the `Value` trait.
+pub struct _34; impl Value for _34 { const VALUE: u32 = 34; }
+/// Type representing the value 35 for the `Value` trait.
+pub struct _35; impl Value for _35 { const VALUE: u32 = 35; }
+/// Type representing the value 36 for the `Value` trait.
+pub struct _36; impl Value for _36 { const VALUE: u32 = 36; }
+/// Type representing the value 37 for the `Value` trait.
+pub struct _37; impl Value for _37 { const VALUE: u32 = 37; }
+/// Type representing the value 38 for the `Value` trait.
+pub struct _38; impl Value for _38 { const VALUE: u32 = 38; }
+/// Type representing the value 39 for the `Value` trait.
+pub struct _39; impl Value for _39 { const VALUE: u32 = 39; }
+/// Type representing the value 40 for the `Value` trait.
+pub struct _40; impl Value for _40 { const VALUE: u32 = 40; }
+/// Type representing the value 41 for the `Value` trait.
+pub struct _41; impl Value for _41 { const VALUE: u32 = 41; }
+/// Type representing the value 42 for the `Value` trait.
+pub struct _42; impl Value for _42 { const VALUE: u32 = 42; }
+/// Type representing the value 43 for the `Value` trait.
+pub struct _43; impl Value for _43 { const VALUE: u32 = 43; }
+/// Type representing the value 44 for the `Value` trait.
+pub struct _44; impl Value for _44 { const VALUE: u32 = 44; }
+/// Type representing the value 45 for the `Value` trait.
+pub struct _45; impl Value for _45 { const VALUE: u32 = 45; }
+/// Type representing the value 46 for the `Value` trait.
+pub struct _46; impl Value for _46 { const VALUE: u32 = 46; }
+/// Type representing the value 47 for the `Value` trait.
+pub struct _47; impl Value for _47 { const VALUE: u32 = 47; }
+/// Type representing the value 48 for the `Value` trait.
+pub struct _48; impl Value for _48 { const VALUE: u32 = 48; }
+/// Type representing the value 49 for the `Value` trait.
+pub struct _49; impl Value for _49 { const VALUE: u32 = 49; }
+/// Type representing the value 50 for the `Value` trait.
+pub struct _50; impl Value for _50 { const VALUE: u32 = 50; }
+/// Type representing the value 51 for the `Value` trait.
+pub struct _51; impl Value for _51 { const VALUE: u32 = 51; }
+/// Type representing the value 52 for the `Value` trait.
+pub struct _52; impl Value for _52 { const VALUE: u32 = 52; }
+/// Type representing the value 53 for the `Value` trait.
+pub struct _53; impl Value for _53 { const VALUE: u32 = 53; }
+/// Type representing the value 54 for the `Value` trait.
+pub struct _54; impl Value for _54 { const VALUE: u32 = 54; }
+/// Type representing the value 55 for the `Value` trait.
+pub struct _55; impl Value for _55 { const VALUE: u32 = 55; }
+/// Type representing the value 56 for the `Value` trait.
+pub struct _56; impl Value for _56 { const VALUE: u32 = 56; }
+/// Type representing the value 57 for the `Value` trait.
+pub struct _57; impl Value for _57 { const VALUE: u32 = 57; }
+/// Type representing the value 58 for the `Value` trait.
+pub struct _58; impl Value for _58 { const VALUE: u32 = 58; }
+/// Type representing the value 59 for the `Value` trait.
+pub struct _59; impl Value for _59 { const VALUE: u32 = 59; }
+/// Type representing the value 60 for the `Value` trait.
+pub struct _60; impl Value for _60 { const VALUE: u32 = 60; }
+/// Type representing the value 61 for the `Value` trait.
+pub struct _61; impl Value for _61 { const VALUE: u32 = 61; }
+/// Type representing the value 62 for the `Value` trait.
+pub struct _62; impl Value for _62 { const VALUE: u32 = 62; }
+/// Type representing the value 63 for the `Value` trait.
+pub struct _63; impl Value for _63 { const VALUE: u32 = 63; }
+/// Type representing the value 64 for the `Value` trait.
+pub struct _64; impl Value for _64 { const VALUE: u32 = 64; }
+/// Type representing the value 65 for the `Value` trait.
+pub struct _65; impl Value for _65 { const VALUE: u32 = 65; }
+/// Type representing the value 66 for the `Value` trait.
+pub struct _66; impl Value for _66 { const VALUE: u32 = 66; }
+/// Type representing the value 67 for the `Value` trait.
+pub struct _67; impl Value for _67 { const VALUE: u32 = 67; }
+/// Type representing the value 68 for the `Value` trait.
+pub struct _68; impl Value for _68 { const VALUE: u32 = 68; }
+/// Type representing the value 69 for the `Value` trait.
+pub struct _69; impl Value for _69 { const VALUE: u32 = 69; }
+/// Type representing the value 70 for the `Value` trait.
+pub struct _70; impl Value for _70 { const VALUE: u32 = 70; }
+/// Type representing the value 71 for the `Value` trait.
+pub struct _71; impl Value for _71 { const VALUE: u32 = 71; }
+/// Type representing the value 72 for the `Value` trait.
+pub struct _72; impl Value for _72 { const VALUE: u32 = 72; }
+/// Type representing the value 73 for the `Value` trait.
+pub struct _73; impl Value for _73 { const VALUE: u32 = 73; }
+/// Type representing the value 74 for the `Value` trait.
+pub struct _74; impl Value for _74 { const VALUE: u32 = 74; }
+/// Type representing the value 75 for the `Value` trait.
+pub struct _75; impl Value for _75 { const VALUE: u32 = 75; }
+/// Type representing the value 76 for the `Value` trait.
+pub struct _76; impl Value for _76 { const VALUE: u32 = 76; }
+/// Type representing the value 77 for the `Value` trait.
+pub struct _77; impl Value for _77 { const VALUE: u32 = 77; }
+/// Type representing the value 78 for the `Value` trait.
+pub struct _78; impl Value for _78 { const VALUE: u32 = 78; }
+/// Type representing the value 79 for the `Value` trait.
+pub struct _79; impl Value for _79 { const VALUE: u32 = 79; }
+/// Type representing the value 80 for the `Value` trait.
+pub struct _80; impl Value for _80 { const VALUE: u32 = 80; }
+/// Type representing the value 81 for the `Value` trait.
+pub struct _81; impl Value for _81 { const VALUE: u32 = 81; }
+/// Type representing the value 82 for the `Value` trait.
+pub struct _82; impl Value for _82 { const VALUE: u32 = 82; }
+/// Type representing the value 83 for the `Value` trait.
+pub struct _83; impl Value for _83 { const VALUE: u32 = 83; }
+/// Type representing the value 84 for the `Value` trait.
+pub struct _84; impl Value for _84 { const VALUE: u32 = 84; }
+/// Type representing the value 85 for the `Value` trait.
+pub struct _85; impl Value for _85 { const VALUE: u32 = 85; }
+/// Type representing the value 86 for the `Value` trait.
+pub struct _86; impl Value for _86 { const VALUE: u32 = 86; }
+/// Type representing the value 87 for the `Value` trait.
+pub struct _87; impl Value for _87 { const VALUE: u32 = 87; }
+/// Type representing the value 88 for the `Value` trait.
+pub struct _88; impl Value for _88 { const VALUE: u32 = 88; }
+/// Type representing the value 89 for the `Value` trait.
+pub struct _89; impl Value for _89 { const VALUE: u32 = 89; }
+/// Type representing the value 90 for the `Value` trait.
+pub struct _90; impl Value for _90 { const VALUE: u32 = 90; }
+/// Type representing the value 91 for the `Value` trait.
+pub struct _91; impl Value for _91 { const VALUE: u32 = 91; }
+/// Type representing the value 92 for the `Value` trait.
+pub struct _92; impl Value for _92 { const VALUE: u32 = 92; }
+/// Type representing the value 93 for the `Value` trait.
+pub struct _93; impl Value for _93 { const VALUE: u32 = 93; }
+/// Type representing the value 94 for the `Value` trait.
+pub struct _94; impl Value for _94 { const VALUE: u32 = 94; }
+/// Type representing the value 95 for the `Value` trait.
+pub struct _95; impl Value for _95 { const VALUE: u32 = 95; }
+/// Type representing the value 96 for the `Value` trait.
+pub struct _96; impl Value for _96 { const VALUE: u32 = 96; }
+/// Type representing the value 97 for the `Value` trait.
+pub struct _97; impl Value for _97 { const VALUE: u32 = 97; }
+/// Type representing the value 98 for the `Value` trait.
+pub struct _98; impl Value for _98 { const VALUE: u32 = 98; }
+/// Type representing the value 99 for the `Value` trait.
+pub struct _99; impl Value for _99 { const VALUE: u32 = 99; }
+/// Type representing the value 100 for the `Value` trait.
+pub struct _100; impl Value for _100 { const VALUE: u32 = 100; }
+/// Type representing the value 112 for the `Value` trait.
+pub struct _112; impl Value for _112 { const VALUE: u32 = 112; }
+/// Type representing the value 128 for the `Value` trait.
+pub struct _128; impl Value for _128 { const VALUE: u32 = 128; }
+/// Type representing the value 160 for the `Value` trait.
+pub struct _160; impl Value for _160 { const VALUE: u32 = 160; }
+/// Type representing the value 192 for the `Value` trait.
+pub struct _192; impl Value for _192 { const VALUE: u32 = 192; }
+/// Type representing the value 224 for the `Value` trait.
+pub struct _224; impl Value for _224 { const VALUE: u32 = 224; }
+/// Type representing the value 256 for the `Value` trait.
+pub struct _256; impl Value for _256 { const VALUE: u32 = 256; }
+/// Type representing the value 384 for the `Value` trait.
+pub struct _384; impl Value for _384 { const VALUE: u32 = 384; }
+/// Type representing the value 512 for the `Value` trait.
+pub struct _512; impl Value for _512 { const VALUE: u32 = 512; }
+

substrate/primitives/core/src/uint.rs

@@ -0,0 +1,99 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! An unsigned fixed-size integer.
+
+pub use primitive_types::U256;
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use codec::{Encode, Decode};
+	use substrate_serializer as ser;
+
+	macro_rules! test {
+		($name: ident, $test_name: ident) => {
+			#[test]
+			fn $test_name() {
+				let tests = vec![
+					($name::from(0), "0x0"),
+					($name::from(1), "0x1"),
+					($name::from(2), "0x2"),
+					($name::from(10), "0xa"),
+					($name::from(15), "0xf"),
+					($name::from(15), "0xf"),
+					($name::from(16), "0x10"),
+					($name::from(1_000), "0x3e8"),
+					($name::from(100_000), "0x186a0"),
+					($name::from(u64::max_value()), "0xffffffffffffffff"),
+					($name::from(u64::max_value()) + $name::from(1), "0x10000000000000000"),
+				];
+
+				for (number, expected) in tests {
+					assert_eq!(format!("{:?}", expected), ser::to_string_pretty(&number));
+					assert_eq!(number, ser::from_str(&format!("{:?}", expected)).unwrap());
+				}
+
+				// Invalid examples
+				assert!(ser::from_str::<$name>("\"0x\"").unwrap_err().is_data());
+				assert!(ser::from_str::<$name>("\"0xg\"").unwrap_err().is_data());
+				assert!(ser::from_str::<$name>("\"\"").unwrap_err().is_data());
+				assert!(ser::from_str::<$name>("\"10\"").unwrap_err().is_data());
+				assert!(ser::from_str::<$name>("\"0\"").unwrap_err().is_data());
+			}
+		}
+	}
+
+	test!(U256, test_u256);
+
+	#[test]
+	fn test_u256_codec() {
+		let res1 = vec![120, 0, 0, 0, 0, 0, 0, 0,
+						0, 0, 0, 0, 0, 0, 0, 0,
+						0, 0, 0, 0, 0, 0, 0, 0,
+						0, 0, 0, 0, 0, 0, 0, 0];
+		let res2 = vec![0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+						0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+						0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+						0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff];
+
+		assert_eq!(
+			U256::from(120).encode(),
+			res1);
+		assert_eq!(
+			U256::max_value().encode(),
+			res2);
+		assert_eq!(
+			U256::decode(&mut &res1[..]),
+			Ok(U256::from(120)));
+		assert_eq!(
+			U256::decode(&mut &res2[..]),
+			Ok(U256::max_value()));
+	}
+
+	#[test]
+	fn test_large_values() {
+		assert_eq!(
+			ser::to_string_pretty(&!U256::zero()),
+			"\"0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\""
+		);
+		assert!(
+			ser::from_str::<U256>("\"0x1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\"")
+			.unwrap_err()
+			.is_data()
+		);
+	}
+}

substrate/primitives/core/storage/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "substrate-primitives-storage"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+description = "Storage related primitives"
+
+[dependencies]
+rstd = { package = "sr-std", path = "../../sr-std", default-features = false }
+serde = { version = "1.0.101", optional = true, features = ["derive"] }
+impl-serde = { version = "0.2.3", optional = true }
+substrate-debug-derive = { version = "2.0.0", path = "../debug-derive" }
+
+[features]
+default = [ "std" ]
+std = [ "rstd/std", "serde", "impl-serde" ]

substrate/primitives/core/storage/src/lib.rs

@@ -0,0 +1,158 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Primitive types for storage related stuff.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+#[cfg(feature = "std")]
+use serde::{Serialize, Deserialize};
+use substrate_debug_derive::RuntimeDebug;
+
+use rstd::{vec::Vec, borrow::Cow};
+
+/// Storage key.
+#[derive(PartialEq, Eq, RuntimeDebug)]
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone))]
+pub struct StorageKey(
+	#[cfg_attr(feature = "std", serde(with="impl_serde::serialize"))]
+	pub Vec<u8>,
+);
+
+/// Storage data associated to a [`StorageKey`].
+#[derive(PartialEq, Eq, RuntimeDebug)]
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone))]
+pub struct StorageData(
+	#[cfg_attr(feature = "std", serde(with="impl_serde::serialize"))]
+	pub Vec<u8>,
+);
+
+/// A set of key value pairs for storage.
+#[cfg(feature = "std")]
+pub type StorageOverlay = std::collections::HashMap<Vec<u8>, Vec<u8>>;
+
+/// A set of key value pairs for children storage;
+#[cfg(feature = "std")]
+pub type ChildrenStorageOverlay = std::collections::HashMap<Vec<u8>, StorageOverlay>;
+
+/// Storage change set
+#[derive(RuntimeDebug)]
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize, PartialEq, Eq))]
+#[cfg_attr(feature = "std", serde(rename_all = "camelCase"))]
+pub struct StorageChangeSet<Hash> {
+	/// Block hash
+	pub block: Hash,
+	/// A list of changes
+	pub changes: Vec<(StorageKey, Option<StorageData>)>,
+}
+
+/// List of all well known keys and prefixes in storage.
+pub mod well_known_keys {
+	/// Wasm code of the runtime.
+	///
+	/// Stored as a raw byte vector. Required by substrate.
+	pub const CODE: &'static [u8] = b":code";
+
+	/// Number of wasm linear memory pages required for execution of the runtime.
+	///
+	/// The type of this value is encoded `u64`.
+	pub const HEAP_PAGES: &'static [u8] = b":heappages";
+
+	/// Current extrinsic index (u32) is stored under this key.
+	pub const EXTRINSIC_INDEX: &'static [u8] = b":extrinsic_index";
+
+	/// Changes trie configuration is stored under this key.
+	pub const CHANGES_TRIE_CONFIG: &'static [u8] = b":changes_trie";
+
+	/// Prefix of child storage keys.
+	pub const CHILD_STORAGE_KEY_PREFIX: &'static [u8] = b":child_storage:";
+
+	/// Whether a key is a child storage key.
+	///
+	/// This is convenience function which basically checks if the given `key` starts
+	/// with `CHILD_STORAGE_KEY_PREFIX` and doesn't do anything apart from that.
+	pub fn is_child_storage_key(key: &[u8]) -> bool {
+		// Other code might depend on this, so be careful changing this.
+		key.starts_with(CHILD_STORAGE_KEY_PREFIX)
+	}
+
+	/// Determine whether a child trie key is valid.
+	///
+	/// For now, the only valid child trie keys are those starting with `:child_storage:default:`.
+	///
+	/// `child_trie_root` and `child_delta_trie_root` can panic if invalid value is provided to them.
+	pub fn is_child_trie_key_valid(storage_key: &[u8]) -> bool {
+		let has_right_prefix = storage_key.starts_with(b":child_storage:default:");
+		if has_right_prefix {
+			// This is an attempt to catch a change of `is_child_storage_key`, which
+			// just checks if the key has prefix `:child_storage:` at the moment of writing.
+			debug_assert!(
+				is_child_storage_key(&storage_key),
+				"`is_child_trie_key_valid` is a subset of `is_child_storage_key`",
+			);
+		}
+		has_right_prefix
+	}
+}
+
+/// A wrapper around a child storage key.
+///
+/// This wrapper ensures that the child storage key is correct and properly used. It is
+/// impossible to create an instance of this struct without providing a correct `storage_key`.
+pub struct ChildStorageKey<'a> {
+	storage_key: Cow<'a, [u8]>,
+}
+
+impl<'a> ChildStorageKey<'a> {
+	/// Create new instance of `Self`.
+	fn new(storage_key: Cow<'a, [u8]>) -> Option<Self> {
+		if well_known_keys::is_child_trie_key_valid(&storage_key) {
+			Some(ChildStorageKey { storage_key })
+		} else {
+			None
+		}
+	}
+
+	/// Create a new `ChildStorageKey` from a vector.
+	///
+	/// `storage_key` need to start with `:child_storage:default:`
+	/// See `is_child_trie_key_valid` for more details.
+	pub fn from_vec(key: Vec<u8>) -> Option<Self> {
+		Self::new(Cow::Owned(key))
+	}
+
+	/// Create a new `ChildStorageKey` from a slice.
+	///
+	/// `storage_key` need to start with `:child_storage:default:`
+	/// See `is_child_trie_key_valid` for more details.
+	pub fn from_slice(key: &'a [u8]) -> Option<Self> {
+		Self::new(Cow::Borrowed(key))
+	}
+
+	/// Get access to the byte representation of the storage key.
+	///
+	/// This key is guaranteed to be correct.
+	pub fn as_ref(&self) -> &[u8] {
+		&*self.storage_key
+	}
+
+	/// Destruct this instance into an owned vector that represents the storage key.
+	///
+	/// This key is guaranteed to be correct.
+	pub fn into_owned(self) -> Vec<u8> {
+		self.storage_key.into_owned()
+	}
+}

substrate/primitives/externalities/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "substrate-externalities"
+version = "2.0.0"
+license = "GPL-3.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+primitive-types = { version = "0.6", features = ["codec"] }
+primitives-storage = { package = "substrate-primitives-storage", path = "../core/storage" }
+rstd = { package = "sr-std", path = "../sr-std" }
+environmental = { version = "1.0.2" }

substrate/primitives/externalities/src/extensions.rs

@@ -0,0 +1,137 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Externalities extensions storage.
+//!
+//! Externalities support to register a wide variety custom extensions. The [`Extensions`] provides
+//! some convenience functionality to store and retrieve these extensions.
+//!
+//! It is required that each extension implements the [`Extension`] trait.
+
+use std::{collections::HashMap, any::{Any, TypeId}, ops::DerefMut};
+
+/// Marker trait for types that should be registered as [`Externalities`](crate::Externalities) extension.
+///
+/// As extensions are stored as `Box<Any>`, this trait should give more confidence that the correct
+/// type is registered and requested.
+pub trait Extension: Send + Any {
+	/// Return the extension as `&mut dyn Any`.
+	///
+	/// This is a trick to make the trait type castable into an `Any`.
+	fn as_mut_any(&mut self) -> &mut dyn Any;
+}
+
+/// Macro for declaring an extension that usable with [`Extensions`].
+///
+/// The extension will be an unit wrapper struct that implements [`Extension`], `Deref` and
+/// `DerefMut`. The wrapped type is given by the user.
+///
+/// # Example
+/// ```
+/// # use substrate_externalities::decl_extension;
+/// decl_extension! {
+///     /// Some test extension
+///     struct TestExt(String);
+/// }
+/// ```
+#[macro_export]
+macro_rules! decl_extension {
+	(
+		$( #[ $attr:meta ] )*
+		$vis:vis struct $ext_name:ident ($inner:ty);
+	) => {
+		$( #[ $attr ] )*
+		$vis struct $ext_name (pub $inner);
+
+		impl $crate::Extension for $ext_name {
+			fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+				self
+			}
+		}
+
+		impl std::ops::Deref for $ext_name {
+			type Target = $inner;
+
+			fn deref(&self) -> &Self::Target {
+				&self.0
+			}
+		}
+
+		impl std::ops::DerefMut for $ext_name {
+			fn deref_mut(&mut self) -> &mut Self::Target {
+				&mut self.0
+			}
+		}
+	}
+}
+
+/// Something that provides access to the [`Extensions`] store.
+///
+/// This is a super trait of the [`Externalities`](crate::Externalities).
+pub trait ExtensionStore {
+	/// Tries to find a registered extension by the given `type_id` and returns it as a `&mut dyn Any`.
+	///
+	/// It is advised to use [`ExternalitiesExt::extension`](crate::ExternalitiesExt::extension)
+	/// instead of this function to get type system support and automatic type downcasting.
+	fn extension_by_type_id(&mut self, type_id: TypeId) -> Option<&mut dyn Any>;
+}
+
+/// Stores extensions that should be made available through the externalities.
+#[derive(Default)]
+pub struct Extensions {
+	extensions: HashMap<TypeId, Box<dyn Extension>>,
+}
+
+impl Extensions {
+	/// Create new instance of `Self`.
+	pub fn new() -> Self {
+		Self::default()
+	}
+
+	/// Register the given extension.
+	pub fn register<E: Extension>(&mut self, ext: E) {
+		self.extensions.insert(ext.type_id(), Box::new(ext));
+	}
+
+	/// Return a mutable reference to the requested extension.
+	pub fn get_mut(&mut self, ext_type_id: TypeId) -> Option<&mut dyn Any> {
+		self.extensions.get_mut(&ext_type_id).map(DerefMut::deref_mut).map(Extension::as_mut_any)
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	decl_extension! {
+		struct DummyExt(u32);
+	}
+	decl_extension! {
+		struct DummyExt2(u32);
+	}
+
+	#[test]
+	fn register_and_retrieve_extension() {
+		let mut exts = Extensions::new();
+		exts.register(DummyExt(1));
+		exts.register(DummyExt2(2));
+
+		let ext = exts.get_mut(TypeId::of::<DummyExt>()).expect("Extension is registered");
+		let ext_ty = ext.downcast_mut::<DummyExt>().expect("Downcasting works");
+
+		assert_eq!(ext_ty.0, 1);
+	}
+}

substrate/primitives/externalities/src/lib.rs

@@ -0,0 +1,139 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Substrate externalities abstraction
+//!
+//! The externalities mainly provide access to storage and to registered extensions. Extensions
+//! are for example the keystore or the offchain externalities. These externalities are used to
+//! access the node from the runtime via the runtime interfaces.
+//!
+//! This crate exposes the main [`Externalities`] trait.
+
+use primitive_types::H256;
+
+use std::any::{Any, TypeId};
+
+use primitives_storage::ChildStorageKey;
+
+pub use scope_limited::{set_and_run_with_externalities, with_externalities};
+pub use extensions::{Extension, Extensions, ExtensionStore};
+
+mod extensions;
+mod scope_limited;
+
+/// The Substrate externalities.
+///
+/// Provides access to the storage and to other registered extensions.
+pub trait Externalities: ExtensionStore {
+	/// Read runtime storage.
+	fn storage(&self, key: &[u8]) -> Option<Vec<u8>>;
+
+	/// Get storage value hash. This may be optimized for large values.
+	fn storage_hash(&self, key: &[u8]) -> Option<H256>;
+
+	/// Get child storage value hash. This may be optimized for large values.
+	fn child_storage_hash(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option<H256>;
+
+	/// Read original runtime storage, ignoring any overlayed changes.
+	fn original_storage(&self, key: &[u8]) -> Option<Vec<u8>>;
+
+	/// Read original runtime child storage, ignoring any overlayed changes.
+	fn original_child_storage(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option<Vec<u8>>;
+
+	/// Get original storage value hash, ignoring any overlayed changes.
+	/// This may be optimized for large values.
+	fn original_storage_hash(&self, key: &[u8]) -> Option<H256>;
+
+	/// Get original child storage value hash, ignoring any overlayed changes.
+	/// This may be optimized for large values.
+	fn original_child_storage_hash(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option<H256>;
+
+	/// Read child runtime storage.
+	fn child_storage(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option<Vec<u8>>;
+
+	/// Set storage entry `key` of current contract being called (effective immediately).
+	fn set_storage(&mut self, key: Vec<u8>, value: Vec<u8>) {
+		self.place_storage(key, Some(value));
+	}
+
+	/// Set child storage entry `key` of current contract being called (effective immediately).
+	fn set_child_storage(&mut self, storage_key: ChildStorageKey, key: Vec<u8>, value: Vec<u8>) {
+		self.place_child_storage(storage_key, key, Some(value))
+	}
+
+	/// Clear a storage entry (`key`) of current contract being called (effective immediately).
+	fn clear_storage(&mut self, key: &[u8]) {
+		self.place_storage(key.to_vec(), None);
+	}
+
+	/// Clear a child storage entry (`key`) of current contract being called (effective immediately).
+	fn clear_child_storage(&mut self, storage_key: ChildStorageKey, key: &[u8]) {
+		self.place_child_storage(storage_key, key.to_vec(), None)
+	}
+
+	/// Whether a storage entry exists.
+	fn exists_storage(&self, key: &[u8]) -> bool {
+		self.storage(key).is_some()
+	}
+
+	/// Whether a child storage entry exists.
+	fn exists_child_storage(&self, storage_key: ChildStorageKey, key: &[u8]) -> bool {
+		self.child_storage(storage_key, key).is_some()
+	}
+
+	/// Clear an entire child storage.
+	fn kill_child_storage(&mut self, storage_key: ChildStorageKey);
+
+	/// Clear storage entries which keys are start with the given prefix.
+	fn clear_prefix(&mut self, prefix: &[u8]);
+
+	/// Clear child storage entries which keys are start with the given prefix.
+	fn clear_child_prefix(&mut self, storage_key: ChildStorageKey, prefix: &[u8]);
+
+	/// Set or clear a storage entry (`key`) of current contract being called (effective immediately).
+	fn place_storage(&mut self, key: Vec<u8>, value: Option<Vec<u8>>);
+
+	/// Set or clear a child storage entry. Return whether the operation succeeds.
+	fn place_child_storage(&mut self, storage_key: ChildStorageKey, key: Vec<u8>, value: Option<Vec<u8>>);
+
+	/// Get the identity of the chain.
+	fn chain_id(&self) -> u64;
+
+	/// Get the trie root of the current storage map. This will also update all child storage keys
+	/// in the top-level storage map.
+	fn storage_root(&mut self) -> H256;
+
+	/// Get the trie root of a child storage map. This will also update the value of the child
+	/// storage keys in the top-level storage map.
+	/// If the storage root equals the default hash as defined by the trie, the key in the top-level
+	/// storage map will be removed.
+	fn child_storage_root(&mut self, storage_key: ChildStorageKey) -> Vec<u8>;
+
+	/// Get the change trie root of the current storage overlay at a block with given parent.
+	fn storage_changes_root(&mut self, parent: H256) -> Result<Option<H256>, ()>;
+}
+
+/// Extension for the [`Externalities`] trait.
+pub trait ExternalitiesExt {
+	/// Tries to find a registered extension and returns a mutable reference.
+	fn extension<T: Any + Extension>(&mut self) -> Option<&mut T>;
+}
+
+impl ExternalitiesExt for &mut dyn Externalities {
+	fn extension<T: Any + Extension>(&mut self) -> Option<&mut T> {
+		self.extension_by_type_id(TypeId::of::<T>()).and_then(Any::downcast_mut)
+	}
+}

substrate/primitives/externalities/src/scope_limited.rs

@@ -0,0 +1,37 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Stores the externalities in an `environmental` value to make it scope limited available.
+
+use crate::Externalities;
+
+environmental::environmental!(ext: trait Externalities);
+
+/// Set the given externalities while executing the given closure. To get access to the externalities
+/// while executing the given closure [`with_externalities`] grants access to them. The externalities
+/// are only set for the same thread this function was called from.
+pub fn set_and_run_with_externalities<F, R>(ext: &mut dyn Externalities, f: F) -> R
+	where F: FnOnce() -> R
+{
+	ext::using(ext, f)
+}
+
+/// Execute the given closure with the currently set externalities.
+///
+/// Returns `None` if no externalities are set or `Some(_)` with the result of the closure.
+pub fn with_externalities<F: FnOnce(&mut dyn Externalities) -> R, R>(f: F) -> Option<R> {
+	ext::with(f)
+}

substrate/primitives/finality-grandpa/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "substrate-finality-grandpa-primitives"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+app-crypto = { package = "substrate-application-crypto", path = "../application-crypto", default-features = false }
+codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] }
+rstd = { package = "sr-std", path = "../sr-std", default-features = false }
+serde = { version = "1.0.101", optional = true, features = ["derive"] }
+sr-api = { path = "../sr-api", default-features = false }
+sr-primitives = { path = "../sr-primitives", default-features = false }
+
+[features]
+default = ["std"]
+std = [
+	"app-crypto/std",
+	"codec/std",
+	"rstd/std",
+	"serde",
+	"sr-api/std",
+	"sr-primitives/std",
+]

substrate/primitives/finality-grandpa/src/lib.rs

@@ -0,0 +1,233 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Primitives for GRANDPA integration, suitable for WASM compilation.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+#[cfg(not(feature = "std"))]
+extern crate alloc;
+
+#[cfg(feature = "std")]
+use serde::Serialize;
+use codec::{Encode, Decode, Input, Codec};
+use sr_primitives::{ConsensusEngineId, RuntimeDebug};
+use rstd::borrow::Cow;
+use rstd::vec::Vec;
+
+mod app {
+	use app_crypto::{app_crypto, key_types::GRANDPA, ed25519};
+	app_crypto!(ed25519, GRANDPA);
+}
+
+/// The grandpa crypto scheme defined via the keypair type.
+#[cfg(feature = "std")]
+pub type AuthorityPair = app::Pair;
+
+/// Identity of a Grandpa authority.
+pub type AuthorityId = app::Public;
+
+/// Signature for a Grandpa authority.
+pub type AuthoritySignature = app::Signature;
+
+/// The `ConsensusEngineId` of GRANDPA.
+pub const GRANDPA_ENGINE_ID: ConsensusEngineId = *b"FRNK";
+
+/// The storage key for the current set of weighted Grandpa authorities.
+/// The value stored is an encoded VersionedAuthorityList.
+pub const GRANDPA_AUTHORITIES_KEY: &'static [u8] = b":grandpa_authorities";
+
+/// The weight of an authority.
+pub type AuthorityWeight = u64;
+
+/// The index of an authority.
+pub type AuthorityIndex = u64;
+
+/// The monotonic identifier of a GRANDPA set of authorities.
+pub type SetId = u64;
+
+/// The round indicator.
+pub type RoundNumber = u64;
+
+/// A list of Grandpa authorities with associated weights.
+pub type AuthorityList = Vec<(AuthorityId, AuthorityWeight)>;
+
+/// A scheduled change of authority set.
+#[cfg_attr(feature = "std", derive(Serialize))]
+#[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug)]
+pub struct ScheduledChange<N> {
+	/// The new authorities after the change, along with their respective weights.
+	pub next_authorities: AuthorityList,
+	/// The number of blocks to delay.
+	pub delay: N,
+}
+
+/// An consensus log item for GRANDPA.
+#[cfg_attr(feature = "std", derive(Serialize))]
+#[derive(Decode, Encode, PartialEq, Eq, Clone, RuntimeDebug)]
+pub enum ConsensusLog<N: Codec> {
+	/// Schedule an authority set change.
+	///
+	/// The earliest digest of this type in a single block will be respected,
+	/// provided that there is no `ForcedChange` digest. If there is, then the
+	/// `ForcedChange` will take precedence.
+	///
+	/// No change should be scheduled if one is already and the delay has not
+	/// passed completely.
+	///
+	/// This should be a pure function: i.e. as long as the runtime can interpret
+	/// the digest type it should return the same result regardless of the current
+	/// state.
+	#[codec(index = "1")]
+	ScheduledChange(ScheduledChange<N>),
+	/// Force an authority set change.
+	///
+	/// Forced changes are applied after a delay of _imported_ blocks,
+	/// while pending changes are applied after a delay of _finalized_ blocks.
+	///
+	/// The earliest digest of this type in a single block will be respected,
+	/// with others ignored.
+	///
+	/// No change should be scheduled if one is already and the delay has not
+	/// passed completely.
+	///
+	/// This should be a pure function: i.e. as long as the runtime can interpret
+	/// the digest type it should return the same result regardless of the current
+	/// state.
+	#[codec(index = "2")]
+	ForcedChange(N, ScheduledChange<N>),
+	/// Note that the authority with given index is disabled until the next change.
+	#[codec(index = "3")]
+	OnDisabled(AuthorityIndex),
+	/// A signal to pause the current authority set after the given delay.
+	/// After finalizing the block at _delay_ the authorities should stop voting.
+	#[codec(index = "4")]
+	Pause(N),
+	/// A signal to resume the current authority set after the given delay.
+	/// After authoring the block at _delay_ the authorities should resume voting.
+	#[codec(index = "5")]
+	Resume(N),
+}
+
+impl<N: Codec> ConsensusLog<N> {
+	/// Try to cast the log entry as a contained signal.
+	pub fn try_into_change(self) -> Option<ScheduledChange<N>> {
+		match self {
+			ConsensusLog::ScheduledChange(change) => Some(change),
+			_ => None,
+		}
+	}
+
+	/// Try to cast the log entry as a contained forced signal.
+	pub fn try_into_forced_change(self) -> Option<(N, ScheduledChange<N>)> {
+		match self {
+			ConsensusLog::ForcedChange(median, change) => Some((median, change)),
+			_ => None,
+		}
+	}
+
+	/// Try to cast the log entry as a contained pause signal.
+	pub fn try_into_pause(self) -> Option<N> {
+		match self {
+			ConsensusLog::Pause(delay) => Some(delay),
+			_ => None,
+		}
+	}
+
+	/// Try to cast the log entry as a contained resume signal.
+	pub fn try_into_resume(self) -> Option<N> {
+		match self {
+			ConsensusLog::Resume(delay) => Some(delay),
+			_ => None,
+		}
+	}
+}
+
+/// WASM function call to check for pending changes.
+pub const PENDING_CHANGE_CALL: &str = "grandpa_pending_change";
+/// WASM function call to get current GRANDPA authorities.
+pub const AUTHORITIES_CALL: &str = "grandpa_authorities";
+
+/// The current version of the stored AuthorityList type. The encoding version MUST be updated any
+/// time the AuthorityList type changes.
+const AUTHORITIES_VERISON: u8 = 1;
+
+/// An AuthorityList that is encoded with a version specifier. The encoding version is updated any
+/// time the AuthorityList type changes. This ensures that encodings of different versions of an
+/// AuthorityList are differentiable. Attempting to decode an authority list with an unknown
+/// version will fail.
+#[derive(Default)]
+pub struct VersionedAuthorityList<'a>(Cow<'a, AuthorityList>);
+
+impl<'a> From<AuthorityList> for VersionedAuthorityList<'a> {
+	fn from(authorities: AuthorityList) -> Self {
+		VersionedAuthorityList(Cow::Owned(authorities))
+	}
+}
+
+impl<'a> From<&'a AuthorityList> for VersionedAuthorityList<'a> {
+	fn from(authorities: &'a AuthorityList) -> Self {
+		VersionedAuthorityList(Cow::Borrowed(authorities))
+	}
+}
+
+impl<'a> Into<AuthorityList> for VersionedAuthorityList<'a> {
+	fn into(self) -> AuthorityList {
+		self.0.into_owned()
+	}
+}
+
+impl<'a> Encode for VersionedAuthorityList<'a> {
+	fn size_hint(&self) -> usize {
+		(AUTHORITIES_VERISON, self.0.as_ref()).size_hint()
+	}
+
+	fn using_encoded<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
+		(AUTHORITIES_VERISON, self.0.as_ref()).using_encoded(f)
+	}
+}
+
+impl<'a> Decode for VersionedAuthorityList<'a> {
+	fn decode<I: Input>(value: &mut I) -> Result<Self, codec::Error> {
+		let (version, authorities): (u8, AuthorityList) = Decode::decode(value)?;
+		if version != AUTHORITIES_VERISON {
+			return Err("unknown Grandpa authorities version".into());
+		}
+		Ok(authorities.into())
+	}
+}
+
+sr_api::decl_runtime_apis! {
+	/// APIs for integrating the GRANDPA finality gadget into runtimes.
+	/// This should be implemented on the runtime side.
+	///
+	/// This is primarily used for negotiating authority-set changes for the
+	/// gadget. GRANDPA uses a signaling model of changing authority sets:
+	/// changes should be signaled with a delay of N blocks, and then automatically
+	/// applied in the runtime after those N blocks have passed.
+	///
+	/// The consensus protocol will coordinate the handoff externally.
+	#[api_version(2)]
+	pub trait GrandpaApi {
+		/// Get the current GRANDPA authorities and weights. This should not change except
+		/// for when changes are scheduled and the corresponding delay has passed.
+		///
+		/// When called at block B, it will return the set of authorities that should be
+		/// used to finalize descendants of this block (B+1, B+2, ...). The block B itself
+		/// is finalized by the authorities from block B-1.
+		fn grandpa_authorities() -> AuthorityList;
+	}
+}

substrate/primitives/inherents/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "substrate-inherents"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+parking_lot = { version = "0.9.0", optional = true }
+rstd = { package = "sr-std", path = "../sr-std", default-features = false }
+primitives = { package = "substrate-primitives", path = "../core", default-features = false }
+codec = { package = "parity-scale-codec", version = "1.0.6", default-features = false, features = ["derive"] }
+derive_more = { version = "0.15.0", optional = true }
+
+[features]
+default = [ "std" ]
+std = [
+	"parking_lot",
+	"rstd/std",
+	"codec/std",
+	"primitives/std",
+	"derive_more",
+]

substrate/primitives/inherents/src/lib.rs

@@ -0,0 +1,582 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Provides types and traits for creating and checking inherents.
+//!
+//! Each inherent is added to a produced block. Each runtime decides on which inherents it
+//! wants to attach to its blocks. All data that is required for the runtime to create the inherents
+//! is stored in the `InherentData`. This `InherentData` is constructed by the node and given to
+//! the runtime.
+//!
+//! Types that provide data for inherents, should implement `InherentDataProvider` and need to be
+//! registered at `InherentDataProviders`.
+//!
+//! In the runtime, modules need to implement `ProvideInherent` when they can create and/or check
+//! inherents. By implementing `ProvideInherent`, a module is not enforced to create an inherent.
+//! A module can also just check given inherents. For using a module as inherent provider, it needs
+//! to be registered by the `construct_runtime!` macro. The macro documentation gives more
+//! information on how that is done.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+#![warn(missing_docs)]
+
+use codec::{Encode, Decode};
+
+use rstd::{collections::btree_map::{BTreeMap, IntoIter, Entry}, vec::Vec};
+
+#[cfg(feature = "std")]
+use parking_lot::RwLock;
+
+#[cfg(feature = "std")]
+use std::{sync::Arc, format};
+
+/// An error that can occur within the inherent data system.
+#[derive(Debug, Encode, Decode, derive_more::Display)]
+#[cfg(feature = "std")]
+pub struct Error(String);
+
+#[cfg(feature = "std")]
+impl<T: Into<String>> From<T> for Error {
+	fn from(data: T) -> Error {
+		Self(data.into())
+	}
+}
+
+#[cfg(feature = "std")]
+impl Error {
+	/// Convert this error into a `String`.
+	pub fn into_string(self) -> String {
+		self.0
+	}
+}
+
+/// An error that can occur within the inherent data system.
+#[derive(Encode, primitives::RuntimeDebug)]
+#[cfg(not(feature = "std"))]
+pub struct Error(&'static str);
+
+#[cfg(not(feature = "std"))]
+impl From<&'static str> for Error {
+	fn from(data: &'static str) -> Error {
+		Self(data)
+	}
+}
+
+/// An identifier for an inherent.
+pub type InherentIdentifier = [u8; 8];
+
+/// Inherent data to include in a block.
+#[derive(Clone, Default, Encode, Decode)]
+pub struct InherentData {
+	/// All inherent data encoded with parity-scale-codec and an identifier.
+	data: BTreeMap<InherentIdentifier, Vec<u8>>
+}
+
+impl InherentData {
+	/// Create a new instance.
+	pub fn new() -> Self {
+		Self::default()
+	}
+
+	/// Put data for an inherent into the internal storage.
+	///
+	/// # Return
+	///
+	/// Returns `Ok(())` if the data could be inserted and no data for an inherent with the same
+	/// identifier existed, otherwise an error is returned.
+	///
+	/// Inherent identifiers need to be unique, otherwise decoding of these values will not work!
+	pub fn put_data<I: codec::Encode>(
+		&mut self,
+		identifier: InherentIdentifier,
+		inherent: &I,
+	) -> Result<(), Error> {
+		match self.data.entry(identifier) {
+			Entry::Vacant(entry) => {
+				entry.insert(inherent.encode());
+				Ok(())
+			},
+			Entry::Occupied(_) => {
+				Err("Inherent with same identifier already exists!".into())
+			}
+		}
+	}
+
+	/// Replace the data for an inherent.
+	///
+	/// If it does not exist, the data is just inserted.
+	pub fn replace_data<I: codec::Encode>(
+		&mut self,
+		identifier: InherentIdentifier,
+		inherent: &I,
+	) {
+		self.data.insert(identifier, inherent.encode());
+	}
+
+	/// Returns the data for the requested inherent.
+	///
+	/// # Return
+	///
+	/// - `Ok(Some(I))` if the data could be found and deserialized.
+	/// - `Ok(None)` if the data could not be found.
+	/// - `Err(_)` if the data could be found, but deserialization did not work.
+	pub fn get_data<I: codec::Decode>(
+		&self,
+		identifier: &InherentIdentifier,
+	) -> Result<Option<I>, Error> {
+		match self.data.get(identifier) {
+			Some(inherent) =>
+				I::decode(&mut &inherent[..])
+					.map_err(|_| {
+						"Could not decode requested inherent type!".into()
+					})
+					.map(Some),
+			None => Ok(None)
+		}
+	}
+}
+
+/// The result of checking inherents.
+///
+/// It either returns okay for all checks, stores all occurred errors or just one fatal error.
+///
+/// When a fatal error occurs, all other errors are removed and the implementation needs to
+/// abort checking inherents.
+#[derive(Encode, Decode, Clone)]
+pub struct CheckInherentsResult {
+	/// Did the check succeed?
+	okay: bool,
+	/// Did we encounter a fatal error?
+	fatal_error: bool,
+	/// We use the `InherentData` to store our errors.
+	errors: InherentData,
+}
+
+impl Default for CheckInherentsResult {
+	fn default() -> Self {
+		Self {
+			okay: true,
+			errors: InherentData::new(),
+			fatal_error: false,
+		}
+	}
+}
+
+impl CheckInherentsResult {
+	/// Create a new instance.
+	pub fn new() -> Self {
+		Self::default()
+	}
+
+	/// Put an error into the result.
+	///
+	/// This makes this result resolve to `ok() == false`.
+	///
+	/// # Parameters
+	///
+	/// - identifier - The identifier of the inherent that generated the error.
+	/// - error - The error that will be encoded.
+	pub fn put_error<E: codec::Encode + IsFatalError>(
+		&mut self,
+		identifier: InherentIdentifier,
+		error: &E,
+	) -> Result<(), Error> {
+		// Don't accept any other error
+		if self.fatal_error {
+			return Err("No other errors are accepted after an hard error!".into())
+		}
+
+		if error.is_fatal_error() {
+			// remove the other errors.
+			self.errors.data.clear();
+		}
+
+		self.errors.put_data(identifier, error)?;
+
+		self.okay = false;
+		self.fatal_error = error.is_fatal_error();
+		Ok(())
+	}
+
+	/// Get an error out of the result.
+	///
+	/// # Return
+	///
+	/// - `Ok(Some(I))` if the error could be found and deserialized.
+	/// - `Ok(None)` if the error could not be found.
+	/// - `Err(_)` if the error could be found, but deserialization did not work.
+	pub fn get_error<E: codec::Decode>(
+		&self,
+		identifier: &InherentIdentifier,
+	) -> Result<Option<E>, Error> {
+		self.errors.get_data(identifier)
+	}
+
+	/// Convert into an iterator over all contained errors.
+	pub fn into_errors(self) -> IntoIter<InherentIdentifier, Vec<u8>> {
+		self.errors.data.into_iter()
+	}
+
+	/// Is this result ok?
+	pub fn ok(&self) -> bool {
+		self.okay
+	}
+
+	/// Is this a fatal error?
+	pub fn fatal_error(&self) -> bool {
+		self.fatal_error
+	}
+}
+
+#[cfg(feature = "std")]
+impl PartialEq for CheckInherentsResult {
+	fn eq(&self, other: &Self) -> bool {
+		self.fatal_error == other.fatal_error &&
+		self.okay == other.okay &&
+		self.errors.data == other.errors.data
+	}
+}
+
+/// All `InherentData` providers.
+#[cfg(feature = "std")]
+#[derive(Clone, Default)]
+pub struct InherentDataProviders {
+	providers: Arc<RwLock<Vec<Box<dyn ProvideInherentData + Send + Sync>>>>,
+}
+
+#[cfg(feature = "std")]
+impl InherentDataProviders {
+	/// Create a new instance.
+	pub fn new() -> Self {
+		Self::default()
+	}
+
+	/// Register an `InherentData` provider.
+	///
+	/// The registration order is preserved and this order will also be used when creating the
+	/// inherent data.
+	///
+	/// # Result
+	///
+	/// Will return an error, if a provider with the same identifier already exists.
+	pub fn register_provider<P: ProvideInherentData + Send + Sync +'static>(
+		&self,
+		provider: P,
+	) -> Result<(), Error> {
+		if self.has_provider(&provider.inherent_identifier()) {
+			Err(
+				format!(
+					"Inherent data provider with identifier {:?} already exists!",
+					&provider.inherent_identifier()
+				).into()
+			)
+		} else {
+			provider.on_register(self)?;
+			self.providers.write().push(Box::new(provider));
+			Ok(())
+		}
+	}
+
+	/// Returns if a provider for the given identifier exists.
+	pub fn has_provider(&self, identifier: &InherentIdentifier) -> bool {
+		self.providers.read().iter().any(|p| p.inherent_identifier() == identifier)
+	}
+
+	/// Create inherent data.
+	pub fn create_inherent_data(&self) -> Result<InherentData, Error> {
+		let mut data = InherentData::new();
+		self.providers.read().iter().try_for_each(|p| {
+			p.provide_inherent_data(&mut data)
+				.map_err(|e| format!("Error for `{:?}`: {:?}", p.inherent_identifier(), e))
+		})?;
+		Ok(data)
+	}
+
+	/// Converts a given encoded error into a `String`.
+	///
+	/// Useful if the implementation encouters an error for an identifier it does not know.
+	pub fn error_to_string(&self, identifier: &InherentIdentifier, error: &[u8]) -> String {
+		let res = self.providers.read().iter().filter_map(|p|
+			if p.inherent_identifier() == identifier {
+				Some(
+					p.error_to_string(error)
+						.unwrap_or_else(|| error_to_string_fallback(identifier))
+				)
+			} else {
+				None
+			}
+		).next();
+
+		match res {
+			Some(res) => res,
+			None => format!(
+				"Error while checking inherent of type \"{}\", but this inherent type is unknown.",
+				String::from_utf8_lossy(identifier)
+			)
+		}
+	}
+}
+
+/// Something that provides inherent data.
+#[cfg(feature = "std")]
+pub trait ProvideInherentData {
+	/// Is called when this inherent data provider is registered at the given
+	/// `InherentDataProviders`.
+	fn on_register(&self, _: &InherentDataProviders) -> Result<(), Error> {
+		Ok(())
+	}
+
+	/// The identifier of the inherent for that data will be provided.
+	fn inherent_identifier(&self) -> &'static InherentIdentifier;
+
+	/// Provide inherent data that should be included in a block.
+	///
+	/// The data should be stored in the given `InherentData` structure.
+	fn provide_inherent_data(&self, inherent_data: &mut InherentData) -> Result<(), Error>;
+
+	/// Convert the given encoded error to a string.
+	///
+	/// If the given error could not be decoded, `None` should be returned.
+	fn error_to_string(&self, error: &[u8]) -> Option<String>;
+}
+
+/// A fallback function, if the decoding of an error fails.
+#[cfg(feature = "std")]
+fn error_to_string_fallback(identifier: &InherentIdentifier) -> String {
+	format!(
+		"Error while checking inherent of type \"{}\", but error could not be decoded.",
+		String::from_utf8_lossy(identifier)
+	)
+}
+
+/// Did we encounter a fatal error while checking an inherent?
+///
+/// A fatal error is everything that fails while checking an inherent error, e.g. the inherent
+/// was not found, could not be decoded etc.
+/// Then there are cases where you not want the inherent check to fail, but report that there is an
+/// action required. For example a timestamp of a block is in the future, the timestamp is still
+/// correct, but it is required to verify the block at a later time again and then the inherent
+/// check will succeed.
+pub trait IsFatalError {
+	/// Is this a fatal error?
+	fn is_fatal_error(&self) -> bool;
+}
+
+/// Auxiliary to make any given error resolve to `is_fatal_error() == true`.
+#[derive(Encode)]
+pub struct MakeFatalError<E: codec::Encode>(E);
+
+impl<E: codec::Encode> From<E> for MakeFatalError<E> {
+	fn from(err: E) -> Self {
+		MakeFatalError(err)
+	}
+}
+
+impl<E: codec::Encode> IsFatalError for MakeFatalError<E> {
+	fn is_fatal_error(&self) -> bool {
+		true
+	}
+}
+
+/// A module that provides an inherent and may also verifies it.
+pub trait ProvideInherent {
+	/// The call type of the module.
+	type Call;
+	/// The error returned by `check_inherent`.
+	type Error: codec::Encode + IsFatalError;
+	/// The inherent identifier used by this inherent.
+	const INHERENT_IDENTIFIER: self::InherentIdentifier;
+
+	/// Create an inherent out of the given `InherentData`.
+	fn create_inherent(data: &InherentData) -> Option<Self::Call>;
+
+	/// Check the given inherent if it is valid.
+	/// Checking the inherent is optional and can be omitted.
+	fn check_inherent(_: &Self::Call, _: &InherentData) -> Result<(), Self::Error> {
+		Ok(())
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use codec::{Encode, Decode};
+
+	const TEST_INHERENT_0: InherentIdentifier = *b"testinh0";
+	const TEST_INHERENT_1: InherentIdentifier = *b"testinh1";
+
+	#[derive(Encode)]
+	struct NoFatalError<E: codec::Encode>(E);
+	impl<E: codec::Encode> IsFatalError for NoFatalError<E> {
+		fn is_fatal_error(&self) -> bool {
+			false
+		}
+	}
+
+	#[test]
+	fn inherent_data_encodes_and_decodes() {
+		let inherent_0 = vec![1, 2, 3];
+		let inherent_1: u32 = 7;
+
+		let mut data = InherentData::new();
+		data.put_data(TEST_INHERENT_0, &inherent_0).unwrap();
+		data.put_data(TEST_INHERENT_1, &inherent_1).unwrap();
+
+		let encoded = data.encode();
+
+		let decoded = InherentData::decode(&mut &encoded[..]).unwrap();
+
+		assert_eq!(decoded.get_data::<Vec<u32>>(&TEST_INHERENT_0).unwrap().unwrap(), inherent_0);
+		assert_eq!(decoded.get_data::<u32>(&TEST_INHERENT_1).unwrap().unwrap(), inherent_1);
+	}
+
+	#[test]
+	fn adding_same_inherent_returns_an_error() {
+		let mut data = InherentData::new();
+		data.put_data(TEST_INHERENT_0, &8).unwrap();
+		assert!(data.put_data(TEST_INHERENT_0, &10).is_err());
+	}
+
+	#[derive(Clone)]
+	struct TestInherentDataProvider {
+		registered: Arc<RwLock<bool>>,
+	}
+
+	impl TestInherentDataProvider {
+		fn new() -> Self {
+			let inst = Self {
+				registered: Default::default(),
+			};
+
+			// just make sure
+			assert!(!inst.is_registered());
+
+			inst
+		}
+
+		fn is_registered(&self) -> bool {
+			*self.registered.read()
+		}
+	}
+
+	const ERROR_TO_STRING: &str = "Found error!";
+
+	impl ProvideInherentData for TestInherentDataProvider {
+		fn on_register(&self, _: &InherentDataProviders) -> Result<(), Error> {
+			*self.registered.write() = true;
+			Ok(())
+		}
+
+		fn inherent_identifier(&self) -> &'static InherentIdentifier {
+			&TEST_INHERENT_0
+		}
+
+		fn provide_inherent_data(&self, data: &mut InherentData) -> Result<(), Error> {
+			data.put_data(TEST_INHERENT_0, &42)
+		}
+
+		fn error_to_string(&self, _: &[u8]) -> Option<String> {
+			Some(ERROR_TO_STRING.into())
+		}
+	}
+
+	#[test]
+	fn registering_inherent_provider() {
+		let provider = TestInherentDataProvider::new();
+		let providers = InherentDataProviders::new();
+
+		providers.register_provider(provider.clone()).unwrap();
+		assert!(provider.is_registered());
+		assert!(providers.has_provider(provider.inherent_identifier()));
+
+		// Second time should fail
+		assert!(providers.register_provider(provider.clone()).is_err());
+	}
+
+	#[test]
+	fn create_inherent_data_from_all_providers() {
+		let provider = TestInherentDataProvider::new();
+		let providers = InherentDataProviders::new();
+
+		providers.register_provider(provider.clone()).unwrap();
+		assert!(provider.is_registered());
+
+		let inherent_data = providers.create_inherent_data().unwrap();
+
+		assert_eq!(
+			inherent_data.get_data::<u32>(provider.inherent_identifier()).unwrap().unwrap(),
+			42u32
+		);
+	}
+
+	#[test]
+	fn encoded_error_to_string() {
+		let provider = TestInherentDataProvider::new();
+		let providers = InherentDataProviders::new();
+
+		providers.register_provider(provider.clone()).unwrap();
+		assert!(provider.is_registered());
+
+		assert_eq!(
+			&providers.error_to_string(&TEST_INHERENT_0, &[1, 2]), ERROR_TO_STRING
+		);
+
+		assert!(
+			providers
+				.error_to_string(&TEST_INHERENT_1, &[1, 2])
+				.contains("inherent type is unknown")
+		);
+	}
+
+	#[test]
+	fn check_inherents_result_encodes_and_decodes() {
+		let mut result = CheckInherentsResult::new();
+		assert!(result.ok());
+
+		result.put_error(TEST_INHERENT_0, &NoFatalError(2u32)).unwrap();
+		assert!(!result.ok());
+		assert!(!result.fatal_error());
+
+		let encoded = result.encode();
+
+		let decoded = CheckInherentsResult::decode(&mut &encoded[..]).unwrap();
+
+		assert_eq!(decoded.get_error::<u32>(&TEST_INHERENT_0).unwrap().unwrap(), 2);
+		assert!(!decoded.ok());
+		assert!(!decoded.fatal_error());
+	}
+
+	#[test]
+	fn check_inherents_result_removes_other_errors_on_fatal_error() {
+		let mut result = CheckInherentsResult::new();
+		assert!(result.ok());
+
+		result.put_error(TEST_INHERENT_0, &NoFatalError(2u32)).unwrap();
+		assert!(!result.ok());
+		assert!(!result.fatal_error());
+
+		result.put_error(TEST_INHERENT_1, &MakeFatalError(4u32)).unwrap();
+		assert!(!result.ok());
+		assert!(result.fatal_error());
+
+		assert!(result.put_error(TEST_INHERENT_0, &NoFatalError(5u32)).is_err());
+
+		result.into_errors().for_each(|(i, e)| match i {
+			TEST_INHERENT_1 => assert_eq!(u32::decode(&mut &e[..]).unwrap(), 4),
+			_ => panic!("There should be no other error!"),
+		});
+	}
+}

substrate/primitives/keyring/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "substrate-keyring"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+primitives = { package = "substrate-primitives",  path = "../core" }
+sr-primitives = { path = "../sr-primitives" }
+lazy_static = "1.4.0"
+strum = "0.15.0"
+strum_macros = "0.15.0"

substrate/primitives/keyring/src/ed25519.rs

@@ -0,0 +1,209 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Support code for the runtime. A set of test accounts.
+
+use std::{collections::HashMap, ops::Deref};
+use lazy_static::lazy_static;
+use primitives::{ed25519::{Pair, Public, Signature}, Pair as PairT, Public as PublicT, H256};
+pub use primitives::ed25519;
+use sr_primitives::AccountId32;
+
+/// Set of test accounts.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, strum_macros::Display, strum_macros::EnumIter)]
+pub enum Keyring {
+	Alice,
+	Bob,
+	Charlie,
+	Dave,
+	Eve,
+	Ferdie,
+	One,
+	Two,
+}
+
+impl Keyring {
+	pub fn from_public(who: &Public) -> Option<Keyring> {
+		Self::iter().find(|&k| &Public::from(k) == who)
+	}
+
+	pub fn from_account_id(who: &AccountId32) -> Option<Keyring> {
+		Self::iter().find(|&k| &k.to_account_id() == who)
+	}
+
+	pub fn from_raw_public(who: [u8; 32]) -> Option<Keyring> {
+		Self::from_public(&Public::from_raw(who))
+	}
+
+	pub fn to_raw_public(self) -> [u8; 32] {
+		*Public::from(self).as_array_ref()
+	}
+
+	pub fn from_h256_public(who: H256) -> Option<Keyring> {
+		Self::from_public(&Public::from_raw(who.into()))
+	}
+
+	pub fn to_h256_public(self) -> H256 {
+		Public::from(self).as_array_ref().into()
+	}
+
+	pub fn to_raw_public_vec(self) -> Vec<u8> {
+		Public::from(self).to_raw_vec()
+	}
+
+	pub fn to_account_id(self) -> AccountId32 {
+		self.to_raw_public().into()
+	}
+
+	pub fn sign(self, msg: &[u8]) -> Signature {
+		Pair::from(self).sign(msg)
+	}
+
+	pub fn pair(self) -> Pair {
+		Pair::from_string(&format!("//{}", <&'static str>::from(self)), None)
+			.expect("static values are known good; qed")
+	}
+
+	/// Returns an iterator over all test accounts.
+	pub fn iter() -> impl Iterator<Item=Keyring> {
+		<Self as strum::IntoEnumIterator>::iter()
+	}
+
+	pub fn public(self) -> Public {
+		self.pair().public()
+	}
+
+	pub fn to_seed(self) -> String {
+		format!("//{}", self)
+	}
+}
+
+impl From<Keyring> for &'static str {
+	fn from(k: Keyring) -> Self {
+		match k {
+			Keyring::Alice => "Alice",
+			Keyring::Bob => "Bob",
+			Keyring::Charlie => "Charlie",
+			Keyring::Dave => "Dave",
+			Keyring::Eve => "Eve",
+			Keyring::Ferdie => "Ferdie",
+			Keyring::One => "One",
+			Keyring::Two => "Two",
+		}
+	}
+}
+
+impl From<Keyring> for sr_primitives::MultiSigner {
+	fn from(x: Keyring) -> Self {
+		sr_primitives::MultiSigner::Ed25519(x.into())
+	}
+}
+
+lazy_static! {
+	static ref PRIVATE_KEYS: HashMap<Keyring, Pair> = {
+		Keyring::iter().map(|i| (i, i.pair())).collect()
+	};
+
+	static ref PUBLIC_KEYS: HashMap<Keyring, Public> = {
+		PRIVATE_KEYS.iter().map(|(&name, pair)| (name, pair.public())).collect()
+	};
+}
+
+impl From<Keyring> for Public {
+	fn from(k: Keyring) -> Self {
+		(*PUBLIC_KEYS).get(&k).unwrap().clone()
+	}
+}
+
+impl From<Keyring> for AccountId32 {
+	fn from(k: Keyring) -> Self {
+		k.to_account_id()
+	}
+}
+
+impl From<Keyring> for Pair {
+	fn from(k: Keyring) -> Self {
+		k.pair()
+	}
+}
+
+impl From<Keyring> for [u8; 32] {
+	fn from(k: Keyring) -> Self {
+		*(*PUBLIC_KEYS).get(&k).unwrap().as_array_ref()
+	}
+}
+
+impl From<Keyring> for H256 {
+	fn from(k: Keyring) -> Self {
+		(*PUBLIC_KEYS).get(&k).unwrap().as_array_ref().into()
+	}
+}
+
+impl From<Keyring> for &'static [u8; 32] {
+	fn from(k: Keyring) -> Self {
+		(*PUBLIC_KEYS).get(&k).unwrap().as_array_ref()
+	}
+}
+
+impl AsRef<[u8; 32]> for Keyring {
+	fn as_ref(&self) -> &[u8; 32] {
+		(*PUBLIC_KEYS).get(self).unwrap().as_array_ref()
+	}
+}
+
+impl AsRef<Public> for Keyring {
+	fn as_ref(&self) -> &Public {
+		(*PUBLIC_KEYS).get(self).unwrap()
+	}
+}
+
+impl Deref for Keyring {
+	type Target = [u8; 32];
+	fn deref(&self) -> &[u8; 32] {
+		(*PUBLIC_KEYS).get(self).unwrap().as_array_ref()
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use primitives::{ed25519::Pair, Pair as PairT};
+
+	#[test]
+	fn should_work() {
+		assert!(
+			Pair::verify(
+				&Keyring::Alice.sign(b"I am Alice!"),
+				b"I am Alice!",
+				&Keyring::Alice.public(),
+			)
+		);
+		assert!(
+			!Pair::verify(
+				&Keyring::Alice.sign(b"I am Alice!"),
+				b"I am Bob!",
+				&Keyring::Alice.public(),
+			)
+		);
+		assert!(
+			!Pair::verify(
+				&Keyring::Alice.sign(b"I am Alice!"),
+				b"I am Alice!",
+				&Keyring::Bob.public(),
+			)
+		);
+	}
+}

substrate/primitives/keyring/src/lib.rs

@@ -0,0 +1,36 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Support code for the runtime. A set of test accounts.
+
+/// Test account crypto for sr25519.
+pub mod sr25519;
+
+/// Test account crypto for ed25519.
+pub mod ed25519;
+
+/// Convenience export: Sr25519's Keyring is exposed as `AccountKeyring`,
+/// since it tends to be used for accounts (although it may also be used
+/// by authorities).
+pub use sr25519::Keyring as AccountKeyring;
+
+pub use ed25519::Keyring as Ed25519Keyring;
+pub use sr25519::Keyring as Sr25519Keyring;
+
+pub mod test {
+	/// The keyring for use with accounts when using the test runtime.
+	pub use super::ed25519::Keyring as AccountKeyring;
+}

substrate/primitives/keyring/src/sr25519.rs

@@ -0,0 +1,210 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Support code for the runtime. A set of test accounts.
+
+use std::collections::HashMap;
+use std::ops::Deref;
+use lazy_static::lazy_static;
+use primitives::{sr25519::{Pair, Public, Signature}, Pair as PairT, Public as PublicT, H256};
+pub use primitives::sr25519;
+use sr_primitives::AccountId32;
+
+/// Set of test accounts.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, strum_macros::Display, strum_macros::EnumIter)]
+pub enum Keyring {
+	Alice,
+	Bob,
+	Charlie,
+	Dave,
+	Eve,
+	Ferdie,
+	One,
+	Two,
+}
+
+impl Keyring {
+	pub fn from_public(who: &Public) -> Option<Keyring> {
+		Self::iter().find(|&k| &Public::from(k) == who)
+	}
+
+	pub fn from_account_id(who: &AccountId32) -> Option<Keyring> {
+		Self::iter().find(|&k| &k.to_account_id() == who)
+	}
+
+	pub fn from_raw_public(who: [u8; 32]) -> Option<Keyring> {
+		Self::from_public(&Public::from_raw(who))
+	}
+
+	pub fn to_raw_public(self) -> [u8; 32] {
+		*Public::from(self).as_array_ref()
+	}
+
+	pub fn from_h256_public(who: H256) -> Option<Keyring> {
+		Self::from_public(&Public::from_raw(who.into()))
+	}
+
+	pub fn to_h256_public(self) -> H256 {
+		Public::from(self).as_array_ref().into()
+	}
+
+	pub fn to_raw_public_vec(self) -> Vec<u8> {
+		Public::from(self).to_raw_vec()
+	}
+
+	pub fn to_account_id(self) -> AccountId32 {
+		self.to_raw_public().into()
+	}
+
+	pub fn sign(self, msg: &[u8]) -> Signature {
+		Pair::from(self).sign(msg)
+	}
+
+	pub fn pair(self) -> Pair {
+		Pair::from_string(&format!("//{}", <&'static str>::from(self)), None)
+			.expect("static values are known good; qed")
+	}
+
+	/// Returns an iterator over all test accounts.
+	pub fn iter() -> impl Iterator<Item=Keyring> {
+		<Self as strum::IntoEnumIterator>::iter()
+	}
+
+	pub fn public(self) -> Public {
+		self.pair().public()
+	}
+
+	pub fn to_seed(self) -> String {
+		format!("//{}", self)
+	}
+}
+
+impl From<Keyring> for &'static str {
+	fn from(k: Keyring) -> Self {
+		match k {
+			Keyring::Alice => "Alice",
+			Keyring::Bob => "Bob",
+			Keyring::Charlie => "Charlie",
+			Keyring::Dave => "Dave",
+			Keyring::Eve => "Eve",
+			Keyring::Ferdie => "Ferdie",
+			Keyring::One => "One",
+			Keyring::Two => "Two",
+		}
+	}
+}
+
+impl From<Keyring> for sr_primitives::MultiSigner {
+	fn from(x: Keyring) -> Self {
+		sr_primitives::MultiSigner::Sr25519(x.into())
+	}
+}
+
+lazy_static! {
+	static ref PRIVATE_KEYS: HashMap<Keyring, Pair> = {
+		Keyring::iter().map(|i| (i, i.pair())).collect()
+	};
+
+	static ref PUBLIC_KEYS: HashMap<Keyring, Public> = {
+		PRIVATE_KEYS.iter().map(|(&name, pair)| (name, pair.public())).collect()
+	};
+}
+
+impl From<Keyring> for AccountId32 {
+	fn from(k: Keyring) -> Self {
+		k.to_account_id()
+	}
+}
+
+impl From<Keyring> for Public {
+	fn from(k: Keyring) -> Self {
+		(*PUBLIC_KEYS).get(&k).unwrap().clone()
+	}
+}
+
+impl From<Keyring> for Pair {
+	fn from(k: Keyring) -> Self {
+		k.pair()
+	}
+}
+
+impl From<Keyring> for [u8; 32] {
+	fn from(k: Keyring) -> Self {
+		*(*PUBLIC_KEYS).get(&k).unwrap().as_array_ref()
+	}
+}
+
+impl From<Keyring> for H256 {
+	fn from(k: Keyring) -> Self {
+		(*PUBLIC_KEYS).get(&k).unwrap().as_array_ref().into()
+	}
+}
+
+impl From<Keyring> for &'static [u8; 32] {
+	fn from(k: Keyring) -> Self {
+		(*PUBLIC_KEYS).get(&k).unwrap().as_array_ref()
+	}
+}
+
+impl AsRef<[u8; 32]> for Keyring {
+	fn as_ref(&self) -> &[u8; 32] {
+		(*PUBLIC_KEYS).get(self).unwrap().as_array_ref()
+	}
+}
+
+impl AsRef<Public> for Keyring {
+	fn as_ref(&self) -> &Public {
+		(*PUBLIC_KEYS).get(self).unwrap()
+	}
+}
+
+impl Deref for Keyring {
+	type Target = [u8; 32];
+	fn deref(&self) -> &[u8; 32] {
+		(*PUBLIC_KEYS).get(self).unwrap().as_array_ref()
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use primitives::{sr25519::Pair, Pair as PairT};
+
+	#[test]
+	fn should_work() {
+		assert!(
+			Pair::verify(
+				&Keyring::Alice.sign(b"I am Alice!"),
+				b"I am Alice!",
+				&Keyring::Alice.public(),
+			)
+		);
+		assert!(
+			!Pair::verify(
+				&Keyring::Alice.sign(b"I am Alice!"),
+				b"I am Bob!",
+				&Keyring::Alice.public(),
+			)
+		);
+		assert!(
+			!Pair::verify(
+				&Keyring::Alice.sign(b"I am Alice!"),
+				b"I am Alice!",
+				&Keyring::Bob.public(),
+			)
+		);
+	}
+}

substrate/primitives/offchain/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+description = "Substrate offchain workers primitives"
+name = "substrate-offchain-primitives"
+version = "2.0.0"
+license = "GPL-3.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+sr-api = { path = "../sr-api", default-features = false }
+sr-primitives = {  path = "../sr-primitives", default-features = false }
+
+[features]
+default = ["std"]
+std = [
+	"sr-api/std",
+	"sr-primitives/std"
+]

substrate/primitives/offchain/src/lib.rs

@@ -0,0 +1,34 @@
+// Copyright 2018 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The Offchain Worker runtime api primitives.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+#![warn(missing_docs)]
+
+use sr_primitives::traits::NumberFor;
+
+/// Local Storage Prefix used by the Offchain Worker API to
+pub const STORAGE_PREFIX: &[u8] = b"storage";
+
+sr_api::decl_runtime_apis! {
+	/// The offchain worker api.
+	pub trait OffchainWorkerApi {
+		/// Starts the off-chain task for given block number.
+		#[skip_initialize_block]
+		fn offchain_worker(number: NumberFor<Block>);
+	}
+}

substrate/primitives/panic-handler/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "substrate-panic-handler"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Substrate panic handler."
+edition = "2018"
+
+[dependencies]
+backtrace = "0.3.38"
+log = "0.4.8"

substrate/primitives/panic-handler/src/lib.rs

@@ -0,0 +1,191 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Custom panic hook with bug report link
+//!
+//! This crate provides the [`set`] function, which wraps around [`std::panic::set_hook`] and
+//! sets up a panic hook that prints a backtrace and invites the user to open an issue to the
+//! given URL.
+//!
+//! By default, the panic handler aborts the process by calling [`std::process::exit`]. This can
+//! temporarily be disabled by using an [`AbortGuard`].
+
+use backtrace::Backtrace;
+use std::io::{self, Write};
+use std::marker::PhantomData;
+use std::panic::{self, PanicInfo};
+use std::cell::Cell;
+use std::thread;
+
+thread_local! {
+	static ON_PANIC: Cell<OnPanic> = Cell::new(OnPanic::Abort);
+}
+
+/// Panic action.
+#[derive(Debug, Clone, Copy, PartialEq)]
+enum OnPanic {
+	/// Abort when panic occurs.
+	Abort,
+	/// Unwind when panic occurs.
+	Unwind,
+	/// Always unwind even if someone changes strategy to Abort afterwards.
+	NeverAbort,
+}
+
+/// Set the panic hook.
+///
+/// Calls [`std::panic::set_hook`] to set up the panic hook.
+///
+/// The `bug_url` parameter is an invitation for users to visit that URL to submit a bug report
+/// in the case where a panic happens.
+pub fn set(bug_url: &'static str, version: &str) {
+	panic::set_hook(Box::new({
+		let version = version.to_string();
+		move |c| {
+			panic_hook(c, bug_url, &version)
+		}
+	}));
+}
+
+macro_rules! ABOUT_PANIC {
+	() => ("
+This is a bug. Please report it at:
+
+	{}
+")}
+
+/// Set aborting flag. Returns previous value of the flag.
+fn set_abort(on_panic: OnPanic) -> OnPanic {
+	ON_PANIC.with(|val| {
+		let prev = val.get();
+		match prev {
+			OnPanic::Abort | OnPanic::Unwind => val.set(on_panic),
+			OnPanic::NeverAbort => (),
+		}
+		prev
+	})
+}
+
+/// RAII guard for whether panics in the current thread should unwind or abort.
+///
+/// Sets a thread-local abort flag on construction and reverts to the previous setting when dropped.
+/// Does not implement `Send` on purpose.
+///
+/// > **Note**: Because we restore the previous value when dropped, you are encouraged to leave
+/// > the `AbortGuard` on the stack and let it destroy itself naturally.
+pub struct AbortGuard {
+	/// Value that was in `ABORT` before we created this guard.
+	previous_val: OnPanic,
+	/// Marker so that `AbortGuard` doesn't implement `Send`.
+	_not_send: PhantomData<std::rc::Rc<()>>
+}
+
+impl AbortGuard {
+	/// Create a new guard. While the guard is alive, panics that happen in the current thread will
+	/// unwind the stack (unless another guard is created afterwards).
+	pub fn force_unwind() -> AbortGuard {
+		AbortGuard {
+			previous_val: set_abort(OnPanic::Unwind),
+			_not_send: PhantomData
+		}
+	}
+
+	/// Create a new guard. While the guard is alive, panics that happen in the current thread will
+	/// abort the process (unless another guard is created afterwards).
+	pub fn force_abort() -> AbortGuard {
+		AbortGuard {
+			previous_val: set_abort(OnPanic::Abort),
+			_not_send: PhantomData
+		}
+	}
+
+	/// Create a new guard. While the guard is alive, panics that happen in the current thread will
+	/// **never** abort the process (even if `AbortGuard::force_abort()` guard will be created afterwards).
+	pub fn never_abort() -> AbortGuard {
+		AbortGuard {
+			previous_val: set_abort(OnPanic::NeverAbort),
+			_not_send: PhantomData
+		}
+	}
+}
+
+impl Drop for AbortGuard {
+	fn drop(&mut self) {
+		set_abort(self.previous_val);
+	}
+}
+
+/// Function being called when a panic happens.
+fn panic_hook(info: &PanicInfo, report_url: &'static str, version: &str) {
+	let location = info.location();
+	let file = location.as_ref().map(|l| l.file()).unwrap_or("<unknown>");
+	let line = location.as_ref().map(|l| l.line()).unwrap_or(0);
+
+	let msg = match info.payload().downcast_ref::<&'static str>() {
+		Some(s) => *s,
+		None => match info.payload().downcast_ref::<String>() {
+			Some(s) => &s[..],
+			None => "Box<Any>",
+		}
+	};
+
+	let thread = thread::current();
+	let name = thread.name().unwrap_or("<unnamed>");
+
+	let backtrace = Backtrace::new();
+
+	let mut stderr = io::stderr();
+
+	let _ = writeln!(stderr, "");
+	let _ = writeln!(stderr, "====================");
+	let _ = writeln!(stderr, "");
+	let _ = writeln!(stderr, "Version: {}", version);
+	let _ = writeln!(stderr, "");
+	let _ = writeln!(stderr, "{:?}", backtrace);
+	let _ = writeln!(stderr, "");
+	let _ = writeln!(
+		stderr,
+		"Thread '{}' panicked at '{}', {}:{}",
+		name, msg, file, line
+	);
+
+	let _ = writeln!(stderr, ABOUT_PANIC!(), report_url);
+	ON_PANIC.with(|val| {
+		if val.get() == OnPanic::Abort {
+			::std::process::exit(1);
+		}
+	})
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn does_not_abort() {
+		set("test", "1.2.3");
+		let _guard = AbortGuard::force_unwind();
+		::std::panic::catch_unwind(|| panic!()).ok();
+	}
+
+	#[test]
+	fn does_not_abort_after_never_abort() {
+		set("test", "1.2.3");
+		let _guard = AbortGuard::never_abort();
+		let _guard = AbortGuard::force_abort();
+		std::panic::catch_unwind(|| panic!()).ok();
+	}
+}

substrate/primitives/peerset/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+description = "Connectivity manager based on reputation"
+homepage = "http://parity.io"
+license = "GPL-3.0"
+name = "substrate-peerset"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+futures-preview = "0.3.0-alpha.19"
+libp2p = { version = "0.13.0", default-features = false }
+linked-hash-map = "0.5.2"
+log = "0.4.8"
+lru-cache = "0.1.2"
+serde_json = "1.0.41"
+
+[dev-dependencies]
+rand = "0.7.2"

substrate/primitives/peerset/src/lib.rs

@@ -0,0 +1,652 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Peer Set Manager (PSM). Contains the strategy for choosing which nodes the network should be
+//! connected to.
+
+mod peersstate;
+
+use std::{collections::{HashSet, HashMap}, collections::VecDeque, time::Instant};
+use futures::{prelude::*, channel::mpsc};
+use libp2p::PeerId;
+use log::{debug, error, trace};
+use serde_json::json;
+use std::{pin::Pin, task::Context, task::Poll};
+
+/// We don't accept nodes whose reputation is under this value.
+const BANNED_THRESHOLD: i32 = 82 * (i32::min_value() / 100);
+/// Reputation change for a node when we get disconnected from it.
+const DISCONNECT_REPUTATION_CHANGE: i32 = -10;
+/// Reserved peers group ID
+const RESERVED_NODES: &'static str = "reserved";
+
+#[derive(Debug)]
+enum Action {
+	AddReservedPeer(PeerId),
+	RemoveReservedPeer(PeerId),
+	SetReservedOnly(bool),
+	ReportPeer(PeerId, i32),
+	SetPriorityGroup(String, HashSet<PeerId>),
+	AddToPriorityGroup(String, PeerId),
+	RemoveFromPriorityGroup(String, PeerId),
+}
+
+/// Shared handle to the peer set manager (PSM). Distributed around the code.
+#[derive(Debug, Clone)]
+pub struct PeersetHandle {
+	tx: mpsc::UnboundedSender<Action>,
+}
+
+impl PeersetHandle {
+	/// Adds a new reserved peer. The peerset will make an effort to always remain connected to
+	/// this peer.
+	///
+	/// Has no effect if the node was already a reserved peer.
+	///
+	/// > **Note**: Keep in mind that the networking has to know an address for this node,
+	/// >			otherwise it will not be able to connect to it.
+	pub fn add_reserved_peer(&self, peer_id: PeerId) {
+		let _ = self.tx.unbounded_send(Action::AddReservedPeer(peer_id));
+	}
+
+	/// Remove a previously-added reserved peer.
+	///
+	/// Has no effect if the node was not a reserved peer.
+	pub fn remove_reserved_peer(&self, peer_id: PeerId) {
+		let _ = self.tx.unbounded_send(Action::RemoveReservedPeer(peer_id));
+	}
+
+	/// Sets whether or not the peerset only has connections .
+	pub fn set_reserved_only(&self, reserved: bool) {
+		let _ = self.tx.unbounded_send(Action::SetReservedOnly(reserved));
+	}
+
+	/// Reports an adjustment to the reputation of the given peer.
+	pub fn report_peer(&self, peer_id: PeerId, score_diff: i32) {
+		let _ = self.tx.unbounded_send(Action::ReportPeer(peer_id, score_diff));
+	}
+
+	/// Modify a priority group.
+	pub fn set_priority_group(&self, group_id: String, peers: HashSet<PeerId>) {
+		let _ = self.tx.unbounded_send(Action::SetPriorityGroup(group_id, peers));
+	}
+
+	/// Add a peer to a priority group.
+	pub fn add_to_priority_group(&self, group_id: String, peer_id: PeerId) {
+		let _ = self.tx.unbounded_send(Action::AddToPriorityGroup(group_id, peer_id));
+	}
+
+	/// Remove a peer from a priority group.
+	pub fn remove_from_priority_group(&self, group_id: String, peer_id: PeerId) {
+		let _ = self.tx.unbounded_send(Action::RemoveFromPriorityGroup(group_id, peer_id));
+	}
+}
+
+/// Message that can be sent by the peer set manager (PSM).
+#[derive(Debug, PartialEq)]
+pub enum Message {
+	/// Request to open a connection to the given peer. From the point of view of the PSM, we are
+	/// immediately connected.
+	Connect(PeerId),
+
+	/// Drop the connection to the given peer, or cancel the connection attempt after a `Connect`.
+	Drop(PeerId),
+
+	/// Equivalent to `Connect` for the peer corresponding to this incoming index.
+	Accept(IncomingIndex),
+
+	/// Equivalent to `Drop` for the peer corresponding to this incoming index.
+	Reject(IncomingIndex),
+}
+
+/// Opaque identifier for an incoming connection. Allocated by the network.
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct IncomingIndex(pub u64);
+
+impl From<u64> for IncomingIndex {
+	fn from(val: u64) -> IncomingIndex {
+		IncomingIndex(val)
+	}
+}
+
+/// Configuration to pass when creating the peer set manager.
+#[derive(Debug)]
+pub struct PeersetConfig {
+	/// Maximum number of ingoing links to peers.
+	pub in_peers: u32,
+
+	/// Maximum number of outgoing links to peers.
+	pub out_peers: u32,
+
+	/// List of bootstrap nodes to initialize the peer with.
+	///
+	/// > **Note**: Keep in mind that the networking has to know an address for these nodes,
+	/// >			otherwise it will not be able to connect to them.
+	pub bootnodes: Vec<PeerId>,
+
+	/// If true, we only accept reserved nodes.
+	pub reserved_only: bool,
+
+	/// List of nodes that we should always be connected to.
+	///
+	/// > **Note**: Keep in mind that the networking has to know an address for these nodes,
+	/// >			otherwise it will not be able to connect to them.
+	pub reserved_nodes: Vec<PeerId>,
+}
+
+/// Side of the peer set manager owned by the network. In other words, the "receiving" side.
+///
+/// Implements the `Stream` trait and can be polled for messages. The `Stream` never ends and never
+/// errors.
+#[derive(Debug)]
+pub struct Peerset {
+	data: peersstate::PeersState,
+	/// If true, we only accept reserved nodes.
+	reserved_only: bool,
+	/// Receiver for messages from the `PeersetHandle` and from `tx`.
+	rx: mpsc::UnboundedReceiver<Action>,
+	/// Sending side of `rx`.
+	tx: mpsc::UnboundedSender<Action>,
+	/// Queue of messages to be emitted when the `Peerset` is polled.
+	message_queue: VecDeque<Message>,
+	/// When the `Peerset` was created.
+	created: Instant,
+	/// Last time when we updated the reputations of connected nodes.
+	latest_time_update: Instant,
+}
+
+impl Peerset {
+	/// Builds a new peerset from the given configuration.
+	pub fn from_config(config: PeersetConfig) -> (Peerset, PeersetHandle) {
+		let (tx, rx) = mpsc::unbounded();
+
+		let handle = PeersetHandle {
+			tx: tx.clone(),
+		};
+
+		let mut peerset = Peerset {
+			data: peersstate::PeersState::new(config.in_peers, config.out_peers, config.reserved_only),
+			tx,
+			rx,
+			reserved_only: config.reserved_only,
+			message_queue: VecDeque::new(),
+			created: Instant::now(),
+			latest_time_update: Instant::now(),
+		};
+
+		peerset.data.set_priority_group(RESERVED_NODES, config.reserved_nodes.into_iter().collect());
+		for peer_id in config.bootnodes {
+			if let peersstate::Peer::Unknown(entry) = peerset.data.peer(&peer_id) {
+				entry.discover();
+			} else {
+				debug!(target: "peerset", "Duplicate bootnode in config: {:?}", peer_id);
+			}
+		}
+
+		peerset.alloc_slots();
+		(peerset, handle)
+	}
+
+	fn on_add_reserved_peer(&mut self, peer_id: PeerId) {
+		let mut reserved = self.data.get_priority_group(RESERVED_NODES).unwrap_or_default();
+		reserved.insert(peer_id);
+		self.data.set_priority_group(RESERVED_NODES, reserved);
+		self.alloc_slots();
+	}
+
+	fn on_remove_reserved_peer(&mut self, peer_id: PeerId) {
+		let mut reserved = self.data.get_priority_group(RESERVED_NODES).unwrap_or_default();
+		reserved.remove(&peer_id);
+		self.data.set_priority_group(RESERVED_NODES, reserved);
+		match self.data.peer(&peer_id) {
+			peersstate::Peer::Connected(peer) => {
+				if self.reserved_only {
+					peer.disconnect();
+					self.message_queue.push_back(Message::Drop(peer_id));
+				}
+			}
+			peersstate::Peer::NotConnected(_) => {},
+			peersstate::Peer::Unknown(_) => {},
+		}
+	}
+
+	fn on_set_reserved_only(&mut self, reserved_only: bool) {
+		self.reserved_only = reserved_only;
+		self.data.set_priority_only(reserved_only);
+
+		if self.reserved_only {
+			// Disconnect non-reserved nodes.
+			let reserved = self.data.get_priority_group(RESERVED_NODES).unwrap_or_default();
+			for peer_id in self.data.connected_peers().cloned().collect::<Vec<_>>().into_iter() {
+				let peer = self.data.peer(&peer_id).into_connected()
+					.expect("We are enumerating connected peers, therefore the peer is connected; qed");
+				if !reserved.contains(&peer_id) {
+					peer.disconnect();
+					self.message_queue.push_back(Message::Drop(peer_id));
+				}
+			}
+		} else {
+			self.alloc_slots();
+		}
+	}
+
+	fn on_set_priority_group(&mut self, group_id: &str, peers: HashSet<PeerId>) {
+		self.data.set_priority_group(group_id, peers);
+		self.alloc_slots();
+	}
+
+	fn on_add_to_priority_group(&mut self, group_id: &str, peer_id: PeerId) {
+		self.data.add_to_priority_group(group_id, peer_id);
+		self.alloc_slots();
+	}
+
+	fn on_remove_from_priority_group(&mut self, group_id: &str, peer_id: PeerId) {
+		self.data.remove_from_priority_group(group_id, &peer_id);
+		self.alloc_slots();
+	}
+
+	fn on_report_peer(&mut self, peer_id: PeerId, score_diff: i32) {
+		// We want reputations to be up-to-date before adjusting them.
+		self.update_time();
+
+		match self.data.peer(&peer_id) {
+			peersstate::Peer::Connected(mut peer) => {
+				peer.add_reputation(score_diff);
+				if peer.reputation() < BANNED_THRESHOLD {
+					peer.disconnect();
+					self.message_queue.push_back(Message::Drop(peer_id));
+				}
+			},
+			peersstate::Peer::NotConnected(mut peer) => peer.add_reputation(score_diff),
+			peersstate::Peer::Unknown(peer) => peer.discover().add_reputation(score_diff),
+		}
+	}
+
+	/// Updates the value of `self.latest_time_update` and performs all the updates that happen
+	/// over time, such as reputation increases for staying connected.
+	fn update_time(&mut self) {
+		// We basically do `(now - self.latest_update).as_secs()`, except that by the way we do it
+		// we know that we're not going to miss seconds because of rounding to integers.
+		let secs_diff = {
+			let now = Instant::now();
+			let elapsed_latest = self.latest_time_update - self.created;
+			let elapsed_now = now - self.created;
+			self.latest_time_update = now;
+			elapsed_now.as_secs() - elapsed_latest.as_secs()
+		};
+
+		// For each elapsed second, move the node reputation towards zero.
+		// If we multiply each second the reputation by `k` (where `k` is between 0 and 1), it
+		// takes `ln(0.5) / ln(k)` seconds to reduce the reputation by half. Use this formula to
+		// empirically determine a value of `k` that looks correct.
+		for _ in 0..secs_diff {
+			for peer in self.data.peers().cloned().collect::<Vec<_>>() {
+				// We use `k = 0.98`, so we divide by `50`. With that value, it takes 34.3 seconds
+				// to reduce the reputation by half.
+				fn reput_tick(reput: i32) -> i32 {
+					let mut diff = reput / 50;
+					if diff == 0 && reput < 0 {
+						diff = -1;
+					} else if diff == 0 && reput > 0 {
+						diff = 1;
+					}
+					reput.saturating_sub(diff)
+				}
+				match self.data.peer(&peer) {
+					peersstate::Peer::Connected(mut peer) =>
+						peer.set_reputation(reput_tick(peer.reputation())),
+					peersstate::Peer::NotConnected(mut peer) =>
+						peer.set_reputation(reput_tick(peer.reputation())),
+					peersstate::Peer::Unknown(_) => unreachable!("We iterate over known peers; qed")
+				}
+			}
+		}
+	}
+
+	/// Try to fill available out slots with nodes.
+	fn alloc_slots(&mut self) {
+		self.update_time();
+
+		// Try to grab the next node to attempt to connect to.
+		while let Some(next) = {
+			if self.reserved_only {
+				self.data.priority_not_connected_peer_from_group(RESERVED_NODES)
+			} else {
+				self.data.priority_not_connected_peer()
+			}
+		} {
+			match next.try_outgoing() {
+				Ok(conn) => self.message_queue.push_back(Message::Connect(conn.into_peer_id())),
+				Err(_) => break,	// No more slots available.
+			}
+		}
+
+		loop {
+			if self.reserved_only {
+				break
+			}
+
+			// Try to grab the next node to attempt to connect to.
+			let next = match self.data.highest_not_connected_peer() {
+				Some(p) => p,
+				None => break,	// No known node to add.
+			};
+
+			// Don't connect to nodes with an abysmal reputation.
+			if next.reputation() < BANNED_THRESHOLD {
+				break;
+			}
+
+			match next.try_outgoing() {
+				Ok(conn) => self.message_queue.push_back(Message::Connect(conn.into_peer_id())),
+				Err(_) => break,	// No more slots available.
+			}
+		}
+	}
+
+	/// Indicate that we received an incoming connection. Must be answered either with
+	/// a corresponding `Accept` or `Reject`, except if we were already connected to this peer.
+	///
+	/// Note that this mechanism is orthogonal to `Connect`/`Drop`. Accepting an incoming
+	/// connection implicitly means `Connect`, but incoming connections aren't cancelled by
+	/// `dropped`.
+	///
+	// Implementation note: because of concurrency issues, it is possible that we push a `Connect`
+	// message to the output channel with a `PeerId`, and that `incoming` gets called with the same
+	// `PeerId` before that message has been read by the user. In this situation we must not answer.
+	pub fn incoming(&mut self, peer_id: PeerId, index: IncomingIndex) {
+		trace!(target: "peerset", "Incoming {:?}", peer_id);
+		self.update_time();
+
+		let not_connected = match self.data.peer(&peer_id) {
+			// If we're already connected, don't answer, as the docs mention.
+			peersstate::Peer::Connected(_) => return,
+			peersstate::Peer::NotConnected(entry) => entry,
+			peersstate::Peer::Unknown(entry) => entry.discover(),
+		};
+
+		if not_connected.reputation() < BANNED_THRESHOLD {
+			self.message_queue.push_back(Message::Reject(index));
+			return
+		}
+
+		match not_connected.try_accept_incoming() {
+			Ok(_) => self.message_queue.push_back(Message::Accept(index)),
+			Err(_) => self.message_queue.push_back(Message::Reject(index)),
+		}
+	}
+
+	/// Indicate that we dropped an active connection with a peer, or that we failed to connect.
+	///
+	/// Must only be called after the PSM has either generated a `Connect` message with this
+	/// `PeerId`, or accepted an incoming connection with this `PeerId`.
+	pub fn dropped(&mut self, peer_id: PeerId) {
+		trace!(target: "peerset", "Dropping {:?}", peer_id);
+
+		// We want reputations to be up-to-date before adjusting them.
+		self.update_time();
+
+		match self.data.peer(&peer_id) {
+			peersstate::Peer::Connected(mut entry) => {
+				// Decrease the node's reputation so that we don't try it again and again and again.
+				entry.add_reputation(DISCONNECT_REPUTATION_CHANGE);
+				entry.disconnect();
+			}
+			peersstate::Peer::NotConnected(_) | peersstate::Peer::Unknown(_) =>
+				error!(target: "peerset", "Received dropped() for non-connected node"),
+		}
+
+		self.alloc_slots();
+	}
+
+	/// Adds discovered peer ids to the PSM.
+	///
+	/// > **Note**: There is no equivalent "expired" message, meaning that it is the responsibility
+	/// >			of the PSM to remove `PeerId`s that fail to dial too often.
+	pub fn discovered<I: IntoIterator<Item = PeerId>>(&mut self, peer_ids: I) {
+		let mut discovered_any = false;
+
+		for peer_id in peer_ids {
+			if let peersstate::Peer::Unknown(entry) = self.data.peer(&peer_id) {
+				entry.discover();
+				discovered_any = true;
+			}
+		}
+
+		if discovered_any {
+			self.alloc_slots();
+		}
+	}
+
+	/// Reports an adjustment to the reputation of the given peer.
+	pub fn report_peer(&mut self, peer_id: PeerId, score_diff: i32) {
+		// We don't immediately perform the adjustments in order to have state consistency. We
+		// don't want the reporting here to take priority over messages sent using the
+		// `PeersetHandle`.
+		let _ = self.tx.unbounded_send(Action::ReportPeer(peer_id, score_diff));
+	}
+
+	/// Produces a JSON object containing the state of the peerset manager, for debugging purposes.
+	pub fn debug_info(&mut self) -> serde_json::Value {
+		self.update_time();
+
+		json!({
+			"nodes": self.data.peers().cloned().collect::<Vec<_>>().into_iter().map(|peer_id| {
+				let state = match self.data.peer(&peer_id) {
+					peersstate::Peer::Connected(entry) => json!({
+						"connected": true,
+						"reputation": entry.reputation()
+					}),
+					peersstate::Peer::NotConnected(entry) => json!({
+						"connected": false,
+						"reputation": entry.reputation()
+					}),
+					peersstate::Peer::Unknown(_) =>
+						unreachable!("We iterate over the known peers; QED")
+				};
+
+				(peer_id.to_base58(), state)
+			}).collect::<HashMap<_, _>>(),
+			"reserved_only": self.reserved_only,
+			"message_queue": self.message_queue.len(),
+		})
+	}
+
+	/// Returns priority group by id.
+	pub fn get_priority_group(&self, group_id: &str) -> Option<HashSet<PeerId>> {
+		self.data.get_priority_group(group_id)
+	}
+}
+
+impl Stream for Peerset {
+	type Item = Message;
+
+	fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
+		loop {
+			if let Some(message) = self.message_queue.pop_front() {
+				return Poll::Ready(Some(message));
+			}
+
+			let action = match Stream::poll_next(Pin::new(&mut self.rx), cx) {
+				Poll::Pending => return Poll::Pending,
+				Poll::Ready(Some(event)) => event,
+				Poll::Ready(None) => return Poll::Pending,
+			};
+
+			match action {
+				Action::AddReservedPeer(peer_id) =>
+					self.on_add_reserved_peer(peer_id),
+				Action::RemoveReservedPeer(peer_id) =>
+					self.on_remove_reserved_peer(peer_id),
+				Action::SetReservedOnly(reserved) =>
+					self.on_set_reserved_only(reserved),
+				Action::ReportPeer(peer_id, score_diff) =>
+					self.on_report_peer(peer_id, score_diff),
+				Action::SetPriorityGroup(group_id, peers) =>
+					self.on_set_priority_group(&group_id, peers),
+				Action::AddToPriorityGroup(group_id, peer_id) =>
+					self.on_add_to_priority_group(&group_id, peer_id),
+				Action::RemoveFromPriorityGroup(group_id, peer_id) =>
+					self.on_remove_from_priority_group(&group_id, peer_id),
+			}
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use libp2p::PeerId;
+	use futures::prelude::*;
+	use super::{PeersetConfig, Peerset, Message, IncomingIndex, BANNED_THRESHOLD};
+	use std::{pin::Pin, task::Poll, thread, time::Duration};
+
+	fn assert_messages(mut peerset: Peerset, messages: Vec<Message>) -> Peerset {
+		for expected_message in messages {
+			let (message, p) = next_message(peerset).expect("expected message");
+			assert_eq!(message, expected_message);
+			peerset = p;
+		}
+		assert!(peerset.message_queue.is_empty(), peerset.message_queue);
+		peerset
+	}
+
+	fn next_message(mut peerset: Peerset) -> Result<(Message, Peerset), ()> {
+		let next = futures::executor::block_on_stream(&mut peerset).next();
+		let message = next.ok_or_else(|| ())?;
+		Ok((message, peerset))
+	}
+
+	#[test]
+	fn test_peerset_add_reserved_peer() {
+		let bootnode = PeerId::random();
+		let reserved_peer = PeerId::random();
+		let reserved_peer2 = PeerId::random();
+		let config = PeersetConfig {
+			in_peers: 0,
+			out_peers: 2,
+			bootnodes: vec![bootnode],
+			reserved_only: true,
+			reserved_nodes: Vec::new(),
+		};
+
+		let (peerset, handle) = Peerset::from_config(config);
+		handle.add_reserved_peer(reserved_peer.clone());
+		handle.add_reserved_peer(reserved_peer2.clone());
+
+		assert_messages(peerset, vec![
+			Message::Connect(reserved_peer),
+			Message::Connect(reserved_peer2)
+		]);
+	}
+
+	#[test]
+	fn test_peerset_incoming() {
+		let bootnode = PeerId::random();
+		let incoming = PeerId::random();
+		let incoming2 = PeerId::random();
+		let incoming3 = PeerId::random();
+		let ii = IncomingIndex(1);
+		let ii2 = IncomingIndex(2);
+		let ii3 = IncomingIndex(3);
+		let ii4 = IncomingIndex(3);
+		let config = PeersetConfig {
+			in_peers: 2,
+			out_peers: 1,
+			bootnodes: vec![bootnode.clone()],
+			reserved_only: false,
+			reserved_nodes: Vec::new(),
+		};
+
+		let (mut peerset, _handle) = Peerset::from_config(config);
+		peerset.incoming(incoming.clone(), ii);
+		peerset.incoming(incoming.clone(), ii4);
+		peerset.incoming(incoming2.clone(), ii2);
+		peerset.incoming(incoming3.clone(), ii3);
+
+		assert_messages(peerset, vec![
+			Message::Connect(bootnode.clone()),
+			Message::Accept(ii),
+			Message::Accept(ii2),
+			Message::Reject(ii3),
+		]);
+	}
+
+	#[test]
+	fn test_peerset_discovered() {
+		let bootnode = PeerId::random();
+		let discovered = PeerId::random();
+		let discovered2 = PeerId::random();
+		let config = PeersetConfig {
+			in_peers: 0,
+			out_peers: 2,
+			bootnodes: vec![bootnode.clone()],
+			reserved_only: false,
+			reserved_nodes: vec![],
+		};
+
+		let (mut peerset, _handle) = Peerset::from_config(config);
+		peerset.discovered(Some(discovered.clone()));
+		peerset.discovered(Some(discovered.clone()));
+		peerset.discovered(Some(discovered2));
+
+		assert_messages(peerset, vec![
+			Message::Connect(bootnode),
+			Message::Connect(discovered),
+		]);
+	}
+
+	#[test]
+	fn test_peerset_banned() {
+		let (mut peerset, handle) = Peerset::from_config(PeersetConfig {
+			in_peers: 25,
+			out_peers: 25,
+			bootnodes: vec![],
+			reserved_only: false,
+			reserved_nodes: vec![],
+		});
+
+		// We ban a node by setting its reputation under the threshold.
+		let peer_id = PeerId::random();
+		handle.report_peer(peer_id.clone(), BANNED_THRESHOLD - 1);
+
+		let fut = futures::future::poll_fn(move |cx| {
+			// We need one polling for the message to be processed.
+			assert_eq!(Stream::poll_next(Pin::new(&mut peerset), cx), Poll::Pending);
+
+			// Check that an incoming connection from that node gets refused.
+			peerset.incoming(peer_id.clone(), IncomingIndex(1));
+			if let Poll::Ready(msg) = Stream::poll_next(Pin::new(&mut peerset), cx) {
+				assert_eq!(msg.unwrap(), Message::Reject(IncomingIndex(1)));
+			} else {
+				panic!()
+			}
+
+			// Wait a bit for the node's reputation to go above the threshold.
+			thread::sleep(Duration::from_millis(1500));
+
+			// Try again. This time the node should be accepted.
+			peerset.incoming(peer_id.clone(), IncomingIndex(2));
+			while let Poll::Ready(msg) = Stream::poll_next(Pin::new(&mut peerset), cx) {
+				assert_eq!(msg.unwrap(), Message::Accept(IncomingIndex(2)));
+			}
+
+			Poll::Ready(())
+		});
+
+		futures::executor::block_on(fut);
+	}
+}
+

substrate/primitives/peerset/src/peersstate.rs

@@ -0,0 +1,720 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Contains the state storage behind the peerset.
+
+use libp2p::PeerId;
+use std::{borrow::Cow, collections::{HashSet, HashMap}};
+use log::warn;
+
+/// State storage behind the peerset.
+///
+/// # Usage
+///
+/// This struct is nothing more but a data structure containing a list of nodes, where each node
+/// has a reputation and is either connected to us or not.
+///
+#[derive(Debug, Clone)]
+pub struct PeersState {
+	/// List of nodes that we know about.
+	///
+	/// > **Note**: This list should really be ordered by decreasing reputation, so that we can
+	/// 			easily select the best node to connect to. As a first draft, however, we don't
+	/// 			sort, to make the logic easier.
+	nodes: HashMap<PeerId, Node>,
+
+	/// Number of non-priority nodes for which the `ConnectionState` is `In`.
+	num_in: u32,
+
+	/// Number of non-priority nodes for which the `ConnectionState` is `In`.
+	num_out: u32,
+
+	/// Maximum allowed number of non-priority nodes for which the `ConnectionState` is `In`.
+	max_in: u32,
+
+	/// Maximum allowed number of non-priority nodes for which the `ConnectionState` is `Out`.
+	max_out: u32,
+
+	/// Priority groups. Each group is identified by a string ID and contains a set of peer IDs.
+	priority_nodes: HashMap<String, HashSet<PeerId>>,
+
+	/// Only allow connections to/from peers in a priority group.
+	priority_only: bool,
+}
+
+/// State of a single node that we know about.
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+struct Node {
+	/// Whether we are connected to this node.
+	connection_state: ConnectionState,
+
+	/// Reputation value of the node, between `i32::min_value` (we hate that node) and
+	/// `i32::max_value` (we love that node).
+	reputation: i32,
+}
+
+impl Default for Node {
+	fn default() -> Node {
+		Node {
+			connection_state: ConnectionState::NotConnected,
+			reputation: 0,
+		}
+	}
+}
+
+/// Whether we are connected to a node.
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+enum ConnectionState {
+	/// We are connected through an ingoing connection.
+	In,
+	/// We are connected through an outgoing connection.
+	Out,
+	/// We are not connected to this node.
+	NotConnected,
+}
+
+impl ConnectionState {
+	/// Returns `true` for `In` and `Out`.
+	fn is_connected(self) -> bool {
+		match self {
+			ConnectionState::In => true,
+			ConnectionState::Out => true,
+			ConnectionState::NotConnected => false,
+		}
+	}
+}
+
+impl PeersState {
+	/// Builds a new empty `PeersState`.
+	pub fn new(in_peers: u32, out_peers: u32, priority_only: bool) -> Self {
+		PeersState {
+			nodes: HashMap::new(),
+			num_in: 0,
+			num_out: 0,
+			max_in: in_peers,
+			max_out: out_peers,
+			priority_nodes: HashMap::new(),
+			priority_only,
+		}
+	}
+
+	/// Returns an object that grants access to the state of a peer.
+	pub fn peer<'a>(&'a mut self, peer_id: &'a PeerId) -> Peer<'a> {
+		match self.nodes.get_mut(peer_id) {
+			None => return Peer::Unknown(UnknownPeer {
+				parent: self,
+				peer_id: Cow::Borrowed(peer_id),
+			}),
+			Some(peer) => {
+				if peer.connection_state.is_connected() {
+					Peer::Connected(ConnectedPeer {
+						state: self,
+						peer_id: Cow::Borrowed(peer_id),
+					})
+				} else {
+					Peer::NotConnected(NotConnectedPeer {
+						state: self,
+						peer_id: Cow::Borrowed(peer_id),
+					})
+				}
+			}
+		}
+	}
+
+	/// Returns the list of all the peers we know of.
+	// Note: this method could theoretically return a `Peer`, but implementing that
+	// isn't simple.
+	pub fn peers(&self) -> impl Iterator<Item = &PeerId> {
+		self.nodes.keys()
+	}
+
+	/// Returns the list of peers we are connected to.
+	// Note: this method could theoretically return a `ConnectedPeer`, but implementing that
+	// isn't simple.
+	pub fn connected_peers(&self) -> impl Iterator<Item = &PeerId> {
+		self.nodes.iter()
+			.filter(|(_, p)| p.connection_state.is_connected())
+			.map(|(p, _)| p)
+	}
+
+	/// Returns the first priority peer that we are not connected to.
+	///
+	/// If multiple nodes are prioritized, which one is returned is unspecified.
+	pub fn priority_not_connected_peer(&mut self) -> Option<NotConnectedPeer> {
+		let id = self.priority_nodes.values()
+			.flatten()
+			.find(|id| self.nodes.get(id).map_or(false, |node| !node.connection_state.is_connected()))
+			.cloned();
+		id.map(move |id| NotConnectedPeer {
+			state: self,
+			peer_id: Cow::Owned(id),
+		})
+	}
+
+	/// Returns the first priority peer that we are not connected to.
+	///
+	/// If multiple nodes are prioritized, which one is returned is unspecified.
+	pub fn priority_not_connected_peer_from_group(&mut self, group_id: &str) -> Option<NotConnectedPeer> {
+		let id = self.priority_nodes.get(group_id)
+			.and_then(|group| group.iter()
+				.find(|id| self.nodes.get(id).map_or(false, |node| !node.connection_state.is_connected()))
+				.cloned());
+		id.map(move |id| NotConnectedPeer {
+			state: self,
+			peer_id: Cow::Owned(id),
+		})
+	}
+
+	/// Returns the peer with the highest reputation and that we are not connected to.
+	///
+	/// If multiple nodes have the same reputation, which one is returned is unspecified.
+	pub fn highest_not_connected_peer(&mut self) -> Option<NotConnectedPeer> {
+		let outcome = self.nodes
+			.iter_mut()
+			.filter(|(_, Node { connection_state, .. })| !connection_state.is_connected())
+			.fold(None::<(&PeerId, &mut Node)>, |mut cur_node, to_try| {
+				if let Some(cur_node) = cur_node.take() {
+					if cur_node.1.reputation >= to_try.1.reputation {
+						return Some(cur_node);
+					}
+				}
+				Some(to_try)
+			})
+			.map(|(peer_id, _)| peer_id.clone());
+
+		if let Some(peer_id) = outcome {
+			Some(NotConnectedPeer {
+				state: self,
+				peer_id: Cow::Owned(peer_id),
+			})
+		} else {
+			None
+		}
+	}
+
+	fn disconnect(&mut self, peer_id: &PeerId) {
+		let is_priority = self.is_priority(peer_id);
+		if let Some(mut node) = self.nodes.get_mut(peer_id) {
+			if !is_priority {
+				match node.connection_state {
+					ConnectionState::In => self.num_in -= 1,
+					ConnectionState::Out => self.num_out -= 1,
+					ConnectionState::NotConnected =>
+						debug_assert!(false, "State inconsistency: disconnecting a disconnected node")
+				}
+			}
+			node.connection_state = ConnectionState::NotConnected;
+		} else {
+			warn!(target: "peerset", "Attempting to disconnect unknown peer {}", peer_id);
+		}
+	}
+
+	/// Sets the peer as connected with an outgoing connection.
+	fn try_outgoing(&mut self, peer_id: &PeerId) -> bool {
+		let is_priority = self.is_priority(peer_id);
+
+		// We are only accepting connections from priority nodes.
+		if !is_priority && self.priority_only {
+			return false;
+		}
+
+		// Note that it is possible for num_out to be strictly superior to the max, in case we were
+		// connected to reserved node then marked them as not reserved.
+		if self.num_out >= self.max_out && !is_priority {
+			return false;
+		}
+
+		if let Some(mut peer) = self.nodes.get_mut(peer_id) {
+			peer.connection_state = ConnectionState::Out;
+			if !is_priority {
+				self.num_out += 1;
+			}
+			return true;
+		}
+		false
+	}
+
+	/// Tries to accept the peer as an incoming connection.
+	///
+	/// If there are enough slots available, switches the node to "connected" and returns `Ok`. If
+	/// the slots are full, the node stays "not connected" and we return `Err`.
+	///
+	/// Note that reserved nodes don't count towards the number of slots.
+	fn try_accept_incoming(&mut self, peer_id: &PeerId) -> bool {
+		let is_priority = self.is_priority(peer_id);
+
+		// We are only accepting connections from priority nodes.
+		if !is_priority && self.priority_only {
+			return false;
+		}
+
+		// Note that it is possible for num_in to be strictly superior to the max, in case we were
+		// connected to reserved node then marked them as not reserved.
+		if self.num_in >= self.max_in && !is_priority {
+			return false;
+		}
+		if let Some(mut peer) = self.nodes.get_mut(peer_id) {
+			peer.connection_state = ConnectionState::In;
+			if !is_priority {
+				self.num_in += 1;
+			}
+			return true;
+		}
+		false
+	}
+
+	/// Sets priority group
+	pub fn set_priority_group(&mut self, group_id: &str, peers: HashSet<PeerId>) {
+		// update slot counters
+		let all_other_groups: HashSet<_> = self.priority_nodes
+			.iter()
+			.filter(|(g, _)| *g != group_id)
+			.flat_map(|(_, id)| id.clone())
+			.collect();
+		let existing_group = self.priority_nodes.remove(group_id).unwrap_or_default();
+		for id in existing_group {
+			// update slots for nodes that are no longer priority
+			if !all_other_groups.contains(&id) {
+				if let Some(peer) = self.nodes.get_mut(&id) {
+					match peer.connection_state {
+						ConnectionState::In => self.num_in += 1,
+						ConnectionState::Out => self.num_out += 1,
+						ConnectionState::NotConnected => {},
+					}
+				}
+			}
+		}
+
+		for id in &peers {
+			// update slots for nodes that become priority
+			if !all_other_groups.contains(&id) {
+				let peer = self.nodes.entry(id.clone()).or_default();
+				match peer.connection_state {
+					ConnectionState::In => self.num_in -= 1,
+					ConnectionState::Out => self.num_out -= 1,
+					ConnectionState::NotConnected => {},
+				}
+			}
+		}
+		self.priority_nodes.insert(group_id.into(), peers);
+	}
+
+	/// Add a peer to a priority group.
+	pub fn add_to_priority_group(&mut self, group_id: &str, peer_id: PeerId) {
+		let mut peers = self.priority_nodes.get(group_id).cloned().unwrap_or_default();
+		peers.insert(peer_id);
+		self.set_priority_group(group_id, peers);
+	}
+
+	/// Remove a peer from a priority group.
+	pub fn remove_from_priority_group(&mut self, group_id: &str, peer_id: &PeerId) {
+		let mut peers = self.priority_nodes.get(group_id).cloned().unwrap_or_default();
+		peers.remove(&peer_id);
+		self.set_priority_group(group_id, peers);
+	}
+
+	/// Get priority group content.
+	pub fn get_priority_group(&self, group_id: &str) -> Option<HashSet<PeerId>> {
+		self.priority_nodes.get(group_id).cloned()
+	}
+
+	/// Set whether to only allow connections to/from peers in a priority group.
+	/// Calling this method does not affect any existing connection, e.g.
+	/// enabling priority only will not disconnect from any non-priority peers
+	/// we are already connected to, only future incoming/outgoing connection
+	/// attempts will be affected.
+	pub fn set_priority_only(&mut self, priority: bool) {
+		self.priority_only = priority;
+	}
+
+	/// Check that node is any priority group.
+	fn is_priority(&self, peer_id: &PeerId) -> bool {
+		self.priority_nodes.iter().any(|(_, group)| group.contains(peer_id))
+	}
+
+	/// Returns the reputation value of the node.
+	fn reputation(&self, peer_id: &PeerId) -> i32 {
+		self.nodes.get(peer_id).map_or(0, |p| p.reputation)
+	}
+
+	/// Sets the reputation of the peer.
+	fn set_reputation(&mut self, peer_id: &PeerId, value: i32) {
+		let node = self.nodes
+			.entry(peer_id.clone())
+			.or_default();
+		node.reputation = value;
+	}
+
+	/// Performs an arithmetic addition on the reputation score of that peer.
+	///
+	/// In case of overflow, the value will be capped.
+	/// If the peer is unknown to us, we insert it and consider that it has a reputation of 0.
+	fn add_reputation(&mut self, peer_id: &PeerId, modifier: i32) {
+		let node = self.nodes
+			.entry(peer_id.clone())
+			.or_default();
+		node.reputation = node.reputation.saturating_add(modifier);
+	}
+}
+
+/// Grants access to the state of a peer in the `PeersState`.
+pub enum Peer<'a> {
+	/// We are connected to this node.
+	Connected(ConnectedPeer<'a>),
+	/// We are not connected to this node.
+	NotConnected(NotConnectedPeer<'a>),
+	/// We have never heard of this node.
+	Unknown(UnknownPeer<'a>),
+}
+
+impl<'a> Peer<'a> {
+	/// If we are the `Connected` variant, returns the inner `ConnectedPeer`. Returns `None`
+	/// otherwise.
+	pub fn into_connected(self) -> Option<ConnectedPeer<'a>> {
+		match self {
+			Peer::Connected(peer) => Some(peer),
+			Peer::NotConnected(_) => None,
+			Peer::Unknown(_) => None,
+		}
+	}
+
+	/// If we are the `Unknown` variant, returns the inner `ConnectedPeer`. Returns `None`
+	/// otherwise.
+	#[cfg(test)]	// Feel free to remove this if this function is needed outside of tests
+	pub fn into_not_connected(self) -> Option<NotConnectedPeer<'a>> {
+		match self {
+			Peer::Connected(_) => None,
+			Peer::NotConnected(peer) => Some(peer),
+			Peer::Unknown(_) => None,
+		}
+	}
+
+	/// If we are the `Unknown` variant, returns the inner `ConnectedPeer`. Returns `None`
+	/// otherwise.
+	#[cfg(test)]	// Feel free to remove this if this function is needed outside of tests
+	pub fn into_unknown(self) -> Option<UnknownPeer<'a>> {
+		match self {
+			Peer::Connected(_) => None,
+			Peer::NotConnected(_) => None,
+			Peer::Unknown(peer) => Some(peer),
+		}
+	}
+}
+
+/// A peer that is connected to us.
+pub struct ConnectedPeer<'a> {
+	state: &'a mut PeersState,
+	peer_id: Cow<'a, PeerId>,
+}
+
+impl<'a> ConnectedPeer<'a> {
+	/// Destroys this `ConnectedPeer` and returns the `PeerId` inside of it.
+	pub fn into_peer_id(self) -> PeerId {
+		self.peer_id.into_owned()
+	}
+
+	/// Switches the peer to "not connected".
+	pub fn disconnect(self) -> NotConnectedPeer<'a> {
+		self.state.disconnect(&self.peer_id);
+		NotConnectedPeer {
+			state: self.state,
+			peer_id: self.peer_id,
+		}
+	}
+
+	/// Returns the reputation value of the node.
+	pub fn reputation(&self) -> i32 {
+		self.state.reputation(&self.peer_id)
+	}
+
+	/// Sets the reputation of the peer.
+	pub fn set_reputation(&mut self, value: i32) {
+		self.state.set_reputation(&self.peer_id, value)
+	}
+
+	/// Performs an arithmetic addition on the reputation score of that peer.
+	///
+	/// In case of overflow, the value will be capped.
+	pub fn add_reputation(&mut self, modifier: i32) {
+		self.state.add_reputation(&self.peer_id, modifier)
+	}
+}
+
+/// A peer that is not connected to us.
+#[derive(Debug)]
+pub struct NotConnectedPeer<'a> {
+	state: &'a mut PeersState,
+	peer_id: Cow<'a, PeerId>,
+}
+
+impl<'a> NotConnectedPeer<'a> {
+	/// Destroys this `NotConnectedPeer` and returns the `PeerId` inside of it.
+	#[cfg(test)]	// Feel free to remove this if this function is needed outside of tests
+	pub fn into_peer_id(self) -> PeerId {
+		self.peer_id.into_owned()
+	}
+
+	/// Tries to set the peer as connected as an outgoing connection.
+	///
+	/// If there are enough slots available, switches the node to "connected" and returns `Ok`. If
+	/// the slots are full, the node stays "not connected" and we return `Err`.
+	///
+	/// Note that priority nodes don't count towards the number of slots.
+	pub fn try_outgoing(self) -> Result<ConnectedPeer<'a>, NotConnectedPeer<'a>> {
+		if self.state.try_outgoing(&self.peer_id) {
+			Ok(ConnectedPeer {
+				state: self.state,
+				peer_id: self.peer_id,
+			})
+		} else {
+			Err(self)
+		}
+	}
+
+	/// Tries to accept the peer as an incoming connection.
+	///
+	/// If there are enough slots available, switches the node to "connected" and returns `Ok`. If
+	/// the slots are full, the node stays "not connected" and we return `Err`.
+	///
+	/// Note that priority nodes don't count towards the number of slots.
+	pub fn try_accept_incoming(self) -> Result<ConnectedPeer<'a>, NotConnectedPeer<'a>> {
+		if self.state.try_accept_incoming(&self.peer_id) {
+			Ok(ConnectedPeer {
+				state: self.state,
+				peer_id: self.peer_id,
+			})
+		} else {
+			Err(self)
+		}
+	}
+
+	/// Returns the reputation value of the node.
+	pub fn reputation(&self) -> i32 {
+		self.state.reputation(&self.peer_id)
+	}
+
+	/// Sets the reputation of the peer.
+	pub fn set_reputation(&mut self, value: i32) {
+		self.state.set_reputation(&self.peer_id, value)
+	}
+
+	/// Performs an arithmetic addition on the reputation score of that peer.
+	///
+	/// In case of overflow, the value will be capped.
+	/// If the peer is unknown to us, we insert it and consider that it has a reputation of 0.
+	pub fn add_reputation(&mut self, modifier: i32) {
+		self.state.add_reputation(&self.peer_id, modifier)
+	}
+}
+
+/// A peer that we have never heard of.
+pub struct UnknownPeer<'a> {
+	parent: &'a mut PeersState,
+	peer_id: Cow<'a, PeerId>,
+}
+
+impl<'a> UnknownPeer<'a> {
+	/// Inserts the peer identity in our list.
+	///
+	/// The node starts with a reputation of 0. You can adjust these default
+	/// values using the `NotConnectedPeer` that this method returns.
+	pub fn discover(self) -> NotConnectedPeer<'a> {
+		self.parent.nodes.insert(self.peer_id.clone().into_owned(), Node {
+			connection_state: ConnectionState::NotConnected,
+			reputation: 0,
+		});
+
+		let state = self.parent;
+		NotConnectedPeer {
+			state,
+			peer_id: self.peer_id,
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::{PeersState, Peer};
+	use libp2p::PeerId;
+
+	#[test]
+	fn full_slots_in() {
+		let mut peers_state = PeersState::new(1, 1, false);
+		let id1 = PeerId::random();
+		let id2 = PeerId::random();
+
+		if let Peer::Unknown(e) = peers_state.peer(&id1) {
+			assert!(e.discover().try_accept_incoming().is_ok());
+		}
+
+		if let Peer::Unknown(e) = peers_state.peer(&id2) {
+			assert!(e.discover().try_accept_incoming().is_err());
+		}
+	}
+
+	#[test]
+	fn priority_node_doesnt_use_slot() {
+		let mut peers_state = PeersState::new(1, 1, false);
+		let id1 = PeerId::random();
+		let id2 = PeerId::random();
+
+		peers_state.set_priority_group("test", vec![id1.clone()].into_iter().collect());
+		if let Peer::NotConnected(p) = peers_state.peer(&id1) {
+			assert!(p.try_accept_incoming().is_ok());
+		} else { panic!() }
+
+		if let Peer::Unknown(e) = peers_state.peer(&id2) {
+			assert!(e.discover().try_accept_incoming().is_ok());
+		} else { panic!() }
+	}
+
+	#[test]
+	fn disconnecting_frees_slot() {
+		let mut peers_state = PeersState::new(1, 1, false);
+		let id1 = PeerId::random();
+		let id2 = PeerId::random();
+
+		assert!(peers_state.peer(&id1).into_unknown().unwrap().discover().try_accept_incoming().is_ok());
+		assert!(peers_state.peer(&id2).into_unknown().unwrap().discover().try_accept_incoming().is_err());
+		peers_state.peer(&id1).into_connected().unwrap().disconnect();
+		assert!(peers_state.peer(&id2).into_not_connected().unwrap().try_accept_incoming().is_ok());
+	}
+
+	#[test]
+	fn priority_not_connected_peer() {
+		let mut peers_state = PeersState::new(25, 25, false);
+		let id1 = PeerId::random();
+		let id2 = PeerId::random();
+
+		assert!(peers_state.priority_not_connected_peer().is_none());
+		peers_state.peer(&id1).into_unknown().unwrap().discover();
+		peers_state.peer(&id2).into_unknown().unwrap().discover();
+
+		assert!(peers_state.priority_not_connected_peer().is_none());
+		peers_state.set_priority_group("test", vec![id1.clone()].into_iter().collect());
+		assert!(peers_state.priority_not_connected_peer().is_some());
+		peers_state.set_priority_group("test", vec![id2.clone(), id2.clone()].into_iter().collect());
+		assert!(peers_state.priority_not_connected_peer().is_some());
+		peers_state.set_priority_group("test", vec![].into_iter().collect());
+		assert!(peers_state.priority_not_connected_peer().is_none());
+	}
+
+	#[test]
+	fn highest_not_connected_peer() {
+		let mut peers_state = PeersState::new(25, 25, false);
+		let id1 = PeerId::random();
+		let id2 = PeerId::random();
+
+		assert!(peers_state.highest_not_connected_peer().is_none());
+		peers_state.peer(&id1).into_unknown().unwrap().discover().set_reputation(50);
+		peers_state.peer(&id2).into_unknown().unwrap().discover().set_reputation(25);
+		assert_eq!(peers_state.highest_not_connected_peer().map(|p| p.into_peer_id()), Some(id1.clone()));
+		peers_state.peer(&id2).into_not_connected().unwrap().set_reputation(75);
+		assert_eq!(peers_state.highest_not_connected_peer().map(|p| p.into_peer_id()), Some(id2.clone()));
+		peers_state.peer(&id2).into_not_connected().unwrap().try_accept_incoming().unwrap();
+		assert_eq!(peers_state.highest_not_connected_peer().map(|p| p.into_peer_id()), Some(id1.clone()));
+		peers_state.peer(&id1).into_not_connected().unwrap().set_reputation(100);
+		peers_state.peer(&id2).into_connected().unwrap().disconnect();
+		assert_eq!(peers_state.highest_not_connected_peer().map(|p| p.into_peer_id()), Some(id1.clone()));
+		peers_state.peer(&id1).into_not_connected().unwrap().set_reputation(-100);
+		assert_eq!(peers_state.highest_not_connected_peer().map(|p| p.into_peer_id()), Some(id2.clone()));
+	}
+
+	#[test]
+	fn disconnect_priority_doesnt_panic() {
+		let mut peers_state = PeersState::new(1, 1, false);
+		let id = PeerId::random();
+		peers_state.set_priority_group("test", vec![id.clone()].into_iter().collect());
+		let peer = peers_state.peer(&id).into_not_connected().unwrap().try_outgoing().unwrap();
+		peer.disconnect();
+	}
+
+	#[test]
+	fn multiple_priority_groups_slot_count() {
+		let mut peers_state = PeersState::new(1, 1, false);
+		let id = PeerId::random();
+
+		if let Peer::Unknown(p) = peers_state.peer(&id) {
+			assert!(p.discover().try_accept_incoming().is_ok());
+		} else { panic!() }
+
+		assert_eq!(peers_state.num_in, 1);
+		peers_state.set_priority_group("test1", vec![id.clone()].into_iter().collect());
+		assert_eq!(peers_state.num_in, 0);
+		peers_state.set_priority_group("test2", vec![id.clone()].into_iter().collect());
+		assert_eq!(peers_state.num_in, 0);
+		peers_state.set_priority_group("test1", vec![].into_iter().collect());
+		assert_eq!(peers_state.num_in, 0);
+		peers_state.set_priority_group("test2", vec![].into_iter().collect());
+		assert_eq!(peers_state.num_in, 1);
+	}
+
+	#[test]
+	fn priority_only_mode_ignores_drops_unknown_nodes() {
+		// test whether connection to/from given peer is allowed
+		let test_connection = |peers_state: &mut PeersState, id| {
+			if let Peer::Unknown(p) = peers_state.peer(id) {
+				p.discover();
+			}
+
+			let incoming = if let Peer::NotConnected(p) = peers_state.peer(id) {
+				p.try_accept_incoming().is_ok()
+			} else {
+				panic!()
+			};
+
+			if incoming {
+				peers_state.peer(id).into_connected().map(|p| p.disconnect());
+			}
+
+			let outgoing = if let Peer::NotConnected(p) = peers_state.peer(id) {
+				p.try_outgoing().is_ok()
+			} else {
+				panic!()
+			};
+
+			if outgoing {
+				peers_state.peer(id).into_connected().map(|p| p.disconnect());
+			}
+
+			incoming || outgoing
+		};
+
+		let mut peers_state = PeersState::new(1, 1, true);
+		let id = PeerId::random();
+
+		// this is an unknown peer and our peer state is set to only allow
+		// priority peers so any connection attempt should be denied.
+		assert!(!test_connection(&mut peers_state, &id));
+
+		// disabling priority only mode should allow the connection to go
+		// through.
+		peers_state.set_priority_only(false);
+		assert!(test_connection(&mut peers_state, &id));
+
+		// re-enabling it we should again deny connections from the peer.
+		peers_state.set_priority_only(true);
+		assert!(!test_connection(&mut peers_state, &id));
+
+		// but if we add the peer to a priority group it should be accepted.
+		peers_state.set_priority_group("TEST_GROUP", vec![id.clone()].into_iter().collect());
+		assert!(test_connection(&mut peers_state, &id));
+
+		// and removing it will cause the connection to once again be denied.
+		peers_state.remove_from_priority_group("TEST_GROUP", &id);
+		assert!(!test_connection(&mut peers_state, &id));
+	}
+}

substrate/primitives/peerset/tests/fuzz.rs

@@ -0,0 +1,138 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use futures::prelude::*;
+use libp2p::PeerId;
+use rand::distributions::{Distribution, Uniform, WeightedIndex};
+use rand::seq::IteratorRandom;
+use std::{collections::HashMap, collections::HashSet, iter, pin::Pin, task::Poll};
+use substrate_peerset::{IncomingIndex, Message, PeersetConfig, Peerset};
+
+#[test]
+fn run() {
+	for _ in 0..50 {
+		test_once();
+	}
+}
+
+fn test_once() {
+	// PRNG to use.
+	let mut rng = rand::thread_rng();
+
+	// Nodes that the peerset knows about.
+	let mut known_nodes = HashSet::<PeerId>::new();
+	// Nodes that we have reserved. Always a subset of `known_nodes`.
+	let mut reserved_nodes = HashSet::<PeerId>::new();
+
+	let (mut peerset, peerset_handle) = Peerset::from_config(PeersetConfig {
+		bootnodes: (0 .. Uniform::new_inclusive(0, 4).sample(&mut rng)).map(|_| {
+			let id = PeerId::random();
+			known_nodes.insert(id.clone());
+			id
+		}).collect(),
+		reserved_nodes: (0 .. Uniform::new_inclusive(0, 2).sample(&mut rng)).map(|_| {
+			let id = PeerId::random();
+			known_nodes.insert(id.clone());
+			reserved_nodes.insert(id.clone());
+			id
+		}).collect(),
+		reserved_only: Uniform::new_inclusive(0, 10).sample(&mut rng) == 0,
+		in_peers: Uniform::new_inclusive(0, 25).sample(&mut rng),
+		out_peers: Uniform::new_inclusive(0, 25).sample(&mut rng),
+	});
+
+	futures::executor::block_on(futures::future::poll_fn(move |cx| {
+		// List of nodes the user of `peerset` assumes it's connected to. Always a subset of
+		// `known_nodes`.
+		let mut connected_nodes = HashSet::<PeerId>::new();
+		// List of nodes the user of `peerset` called `incoming` with and that haven't been
+		// accepted or rejected yet.
+		let mut incoming_nodes = HashMap::<IncomingIndex, PeerId>::new();
+		// Next id for incoming connections.
+		let mut next_incoming_id = IncomingIndex(0);
+
+		// Perform a certain number of actions while checking that the state is consistent. If we
+		// reach the end of the loop, the run has succeeded.
+		for _ in 0 .. 2500 {
+			// Each of these weights corresponds to an action that we may perform.
+			let action_weights = [150, 90, 90, 30, 30, 1, 1, 4, 4];
+			match WeightedIndex::new(&action_weights).unwrap().sample(&mut rng) {
+				// If we generate 0, poll the peerset.
+				0 => match Stream::poll_next(Pin::new(&mut peerset), cx) {
+					Poll::Ready(Some(Message::Connect(id))) => {
+						if let Some(id) = incoming_nodes.iter().find(|(_, v)| **v == id).map(|(&id, _)| id) {
+							incoming_nodes.remove(&id);
+						}
+						assert!(connected_nodes.insert(id));
+					}
+					Poll::Ready(Some(Message::Drop(id))) => { connected_nodes.remove(&id); }
+					Poll::Ready(Some(Message::Accept(n))) =>
+						assert!(connected_nodes.insert(incoming_nodes.remove(&n).unwrap())),
+					Poll::Ready(Some(Message::Reject(n))) =>
+						assert!(!connected_nodes.contains(&incoming_nodes.remove(&n).unwrap())),
+					Poll::Ready(None) => panic!(),
+					Poll::Pending => {}
+				}
+
+				// If we generate 1, discover a new node.
+				1 => {
+					let new_id = PeerId::random();
+					known_nodes.insert(new_id.clone());
+					peerset.discovered(iter::once(new_id));
+				}
+
+				// If we generate 2, adjust a random reputation.
+				2 => if let Some(id) = known_nodes.iter().choose(&mut rng) {
+					let val = Uniform::new_inclusive(i32::min_value(), i32::max_value()).sample(&mut rng);
+					peerset_handle.report_peer(id.clone(), val);
+				}
+
+				// If we generate 3, disconnect from a random node.
+				3 => if let Some(id) = connected_nodes.iter().choose(&mut rng).cloned() {
+					connected_nodes.remove(&id);
+					peerset.dropped(id);
+				}
+
+				// If we generate 4, connect to a random node.
+				4 => if let Some(id) = known_nodes.iter()
+					.filter(|n| incoming_nodes.values().all(|m| m != *n) && !connected_nodes.contains(n))
+					.choose(&mut rng) {
+					peerset.incoming(id.clone(), next_incoming_id.clone());
+					incoming_nodes.insert(next_incoming_id.clone(), id.clone());
+					next_incoming_id.0 += 1;
+				}
+
+				// 5 and 6 are the reserved-only mode.
+				5 => peerset_handle.set_reserved_only(true),
+				6 => peerset_handle.set_reserved_only(false),
+
+				// 7 and 8 are about switching a random node in or out of reserved mode.
+				7 => if let Some(id) = known_nodes.iter().filter(|n| !reserved_nodes.contains(n)).choose(&mut rng) {
+					peerset_handle.add_reserved_peer(id.clone());
+					reserved_nodes.insert(id.clone());
+				}
+				8 => if let Some(id) = reserved_nodes.iter().choose(&mut rng).cloned() {
+					reserved_nodes.remove(&id);
+					peerset_handle.remove_reserved_peer(id);
+				}
+
+				_ => unreachable!()
+			}
+		}
+
+		Poll::Ready(())
+	}));
+}

substrate/primitives/phragmen/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "substrate-phragmen"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+serde = { version = "1.0.101", optional = true, features = ["derive"] }
+rstd = { package = "sr-std", path = "../sr-std", default-features = false }
+sr-primitives = { path = "../../primitives/sr-primitives", default-features = false }
+
+[dev-dependencies]
+runtime-io ={ package = "sr-io", path = "../../primitives/sr-io" }
+support = { package = "paint-support", path = "../../paint/support" }
+rand = "0.7.2"
+
+[features]
+default = ["std"]
+std = [
+	"serde",
+	"rstd/std",
+	"sr-primitives/std",
+]

substrate/primitives/phragmen/benches/phragmen.rs

@@ -0,0 +1,212 @@
+// Copyright 2019 Parity Technologies
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Benchmarks of the phragmen election algorithm.
+//! Note that execution times will not be accurate in an absolute scale, since
+//! - Everything is executed in the context of `TestExternalities`
+//! - Everything is executed in native environment.
+#![cfg(feature = "bench")]
+#![feature(test)]
+
+extern crate test;
+use test::Bencher;
+
+use rand::{self, Rng};
+extern crate substrate_phragmen as phragmen;
+use phragmen::{Support, SupportMap, PhragmenStakedAssignment};
+
+use std::collections::BTreeMap;
+use sr_primitives::traits::{Convert, SaturatedConversion};
+
+const VALIDATORS: u64 = 1000;
+const NOMINATORS: u64 = 10_000;
+const EDGES: u64 = 2;
+const TO_ELECT: usize = 100;
+const STAKE: Balance = 1000;
+
+type Balance = u128;
+type AccountId = u64;
+
+pub struct TestCurrencyToVote;
+impl Convert<Balance, u64> for TestCurrencyToVote {
+	fn convert(x: Balance) -> u64 { x.saturated_into() }
+}
+impl Convert<u128, Balance> for TestCurrencyToVote {
+	fn convert(x: u128) -> Balance { x.saturated_into() }
+}
+
+fn do_phragmen(
+	b: &mut Bencher,
+	num_vals: u64,
+	num_noms: u64,
+	count: usize,
+	votes_per: u64,
+	eq_iters: usize,
+	_eq_tolerance: u128,
+) {
+	assert!(num_vals > votes_per);
+	let rr = |a, b| rand::thread_rng().gen_range(a as usize, b as usize) as Balance;
+
+	// prefix to distinguish the validator and nominator account ranges.
+	let np = 10_000;
+
+	let mut candidates = Vec::with_capacity(num_vals as usize);
+	let mut slashable_balance_of: BTreeMap<AccountId, Balance> = BTreeMap::new();
+
+	(1 ..= num_vals)
+		.for_each(|acc| {
+			candidates.push(acc);
+			slashable_balance_of.insert(acc, STAKE + rr(10, 50));
+		});
+
+	let mut voters = Vec::with_capacity(num_noms as usize);
+	(np ..= (np + num_noms))
+		.for_each(|acc| {
+			let mut stashes_to_vote = candidates.clone();
+			let votes = (0 .. votes_per)
+				.map(|_| {
+					stashes_to_vote.remove(rr(0, stashes_to_vote.len()) as usize)
+				})
+				.collect::<Vec<AccountId>>();
+			voters.push((acc, votes));
+			slashable_balance_of.insert(acc, STAKE + rr(10, 50));
+		});
+
+	let slashable_balance = |who: &AccountId| -> Balance {
+		*slashable_balance_of.get(who).unwrap()
+	};
+
+	b.iter(|| {
+		let r = phragmen::elect::<AccountId, Balance, _, TestCurrencyToVote>(
+			count,
+			1_usize,
+			candidates.clone(),
+			voters.clone(),
+			slashable_balance,
+			true,
+		).unwrap();
+
+		// Do the benchmarking with equalize.
+		if eq_iters > 0 {
+			let elected_stashes = r.winners;
+			let assignments = r.assignments;
+
+			let to_votes = |b: Balance|
+				<TestCurrencyToVote as Convert<Balance, u128>>::convert(b) as u128;
+
+			// Initialize the support of each candidate.
+			let mut supports = <SupportMap<u64>>::new();
+			elected_stashes
+				.iter()
+				.map(|(e, _)| (e, to_votes(slashable_balance(e))))
+				.for_each(|(e, s)| {
+					let item = Support { own: s, total: s, ..Default::default() };
+					supports.insert(e.clone(), item);
+				});
+
+			// build support struct.
+			for (n, assignment) in assignments.iter() {
+				for (c, per_thing) in assignment.iter() {
+					let nominator_stake = to_votes(slashable_balance(n));
+					let other_stake = *per_thing * nominator_stake;
+					if let Some(support) = supports.get_mut(c) {
+						support.total = support.total.saturating_add(other_stake);
+						support.others.push((n.clone(), other_stake));
+					}
+				}
+			}
+
+			let mut staked_assignments
+				: Vec<(AccountId, Vec<PhragmenStakedAssignment<AccountId>>)>
+				= Vec::with_capacity(assignments.len());
+			for (n, assignment) in assignments.iter() {
+				let mut staked_assignment
+					: Vec<PhragmenStakedAssignment<AccountId>>
+					= Vec::with_capacity(assignment.len());
+				for (c, per_thing) in assignment.iter() {
+					let nominator_stake = to_votes(slashable_balance(n));
+					let other_stake = *per_thing * nominator_stake;
+					staked_assignment.push((c.clone(), other_stake));
+				}
+				staked_assignments.push((n.clone(), staked_assignment));
+			}
+
+			let tolerance = 0_u128;
+			let iterations = 2_usize;
+			phragmen::equalize::<_, _, TestCurrencyToVote, _>(
+				staked_assignments,
+				&mut supports,
+				tolerance,
+				iterations,
+				slashable_balance,
+			);
+		}
+	})
+}
+
+macro_rules! phragmen_benches {
+	($($name:ident: $tup:expr,)*) => {
+	$(
+        #[bench]
+        fn $name(b: &mut Bencher) {
+			let (v, n, t, e, eq_iter, eq_tol) = $tup;
+			println!("----------------------");
+			println!(
+				"++ Benchmark: {} Validators // {} Nominators // {} Edges-per-nominator // {} \
+				total edges // electing {} // Equalize: {} iterations -- {} tolerance",
+				v, n, e, e * n, t, eq_iter, eq_tol,
+			);
+			do_phragmen(b, v, n, t, e, eq_iter, eq_tol);
+        }
+    )*
+	}
+}
+
+phragmen_benches! {
+	bench_1_1: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES, 0, 0),
+	bench_1_2: (VALIDATORS*2, NOMINATORS, TO_ELECT, EDGES, 0, 0),
+	bench_1_3: (VALIDATORS*4, NOMINATORS, TO_ELECT, EDGES, 0, 0),
+	bench_1_4: (VALIDATORS*8, NOMINATORS, TO_ELECT, EDGES, 0, 0),
+	bench_1_1_eq: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES, 2, 0),
+	bench_1_2_eq: (VALIDATORS*2, NOMINATORS, TO_ELECT, EDGES, 2, 0),
+	bench_1_3_eq: (VALIDATORS*4, NOMINATORS, TO_ELECT, EDGES, 2, 0),
+	bench_1_4_eq: (VALIDATORS*8, NOMINATORS, TO_ELECT, EDGES, 2, 0),
+
+	bench_0_1: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES, 0, 0),
+	bench_0_2: (VALIDATORS, NOMINATORS, TO_ELECT * 4, EDGES, 0, 0),
+	bench_0_3: (VALIDATORS, NOMINATORS, TO_ELECT * 8, EDGES, 0, 0),
+	bench_0_4: (VALIDATORS, NOMINATORS, TO_ELECT * 16, EDGES , 0, 0),
+	bench_0_1_eq: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES, 2, 0),
+	bench_0_2_eq: (VALIDATORS, NOMINATORS, TO_ELECT * 4, EDGES, 2, 0),
+	bench_0_3_eq: (VALIDATORS, NOMINATORS, TO_ELECT * 8, EDGES, 2, 0),
+	bench_0_4_eq: (VALIDATORS, NOMINATORS, TO_ELECT * 16, EDGES , 2, 0),
+
+	bench_2_1: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES, 0, 0),
+	bench_2_2: (VALIDATORS, NOMINATORS*2, TO_ELECT, EDGES, 0, 0),
+	bench_2_3: (VALIDATORS, NOMINATORS*4, TO_ELECT, EDGES, 0, 0),
+	bench_2_4: (VALIDATORS, NOMINATORS*8, TO_ELECT, EDGES, 0, 0),
+	bench_2_1_eq: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES, 2, 0),
+	bench_2_2_eq: (VALIDATORS, NOMINATORS*2, TO_ELECT, EDGES, 2, 0),
+	bench_2_3_eq: (VALIDATORS, NOMINATORS*4, TO_ELECT, EDGES, 2, 0),
+	bench_2_4_eq: (VALIDATORS, NOMINATORS*8, TO_ELECT, EDGES, 2, 0),
+
+	bench_3_1: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES, 0, 0 ),
+	bench_3_2: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES*2, 0, 0),
+	bench_3_3: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES*4, 0, 0),
+	bench_3_4: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES*8, 0, 0),
+	bench_3_1_eq: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES, 2, 0),
+	bench_3_2_eq: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES*2, 2, 0),
+	bench_3_3_eq: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES*4, 2, 0),
+	bench_3_4_eq: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES*8, 2, 0),
+}

substrate/primitives/phragmen/src/lib.rs

@@ -0,0 +1,535 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Rust implementation of the Phragmén election algorithm. This is used in several SRML modules to
+//! optimally distribute the weight of a set of voters among an elected set of candidates. In the
+//! context of staking this is mapped to validators and nominators.
+//!
+//! The algorithm has two phases:
+//!   - Sequential phragmen: performed in [`elect`] function which is first pass of the distribution
+//!     The results are not optimal but the execution time is less.
+//!   - Equalize post-processing: tries to further distribute the weight fairly among candidates.
+//!     Incurs more execution time.
+//!
+//! The main objective of the assignments done by phragmen is to maximize the minimum backed
+//! candidate in the elected set.
+//!
+//! Reference implementation: https://github.com/w3f/consensus
+//! Further details:
+//! https://research.web3.foundation/en/latest/polkadot/NPoS/4.%20Sequential%20Phragm%C3%A9n%E2%80%99s%20method/
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+use rstd::{prelude::*, collections::btree_map::BTreeMap};
+use sr_primitives::RuntimeDebug;
+use sr_primitives::{helpers_128bit::multiply_by_rational, Perbill, Rational128};
+use sr_primitives::traits::{Zero, Convert, Member, SimpleArithmetic, Saturating, Bounded};
+
+mod mock;
+mod tests;
+
+/// A type in which performing operations on balances and stakes of candidates and voters are safe.
+///
+/// This module's functions expect a `Convert` type to convert all balances to u64. Hence, u128 is
+/// a safe type for arithmetic operations over them.
+///
+/// Balance types converted to `ExtendedBalance` are referred to as `Votes`.
+pub type ExtendedBalance = u128;
+
+/// The denominator used for loads. Since votes are collected as u64, the smallest ratio that we
+/// might collect is `1/approval_stake` where approval stake is the sum of votes. Hence, some number
+/// bigger than u64::max_value() is needed. For maximum accuracy we simply use u128;
+const DEN: u128 = u128::max_value();
+
+/// A candidate entity for phragmen election.
+#[derive(Clone, Default, RuntimeDebug)]
+pub struct Candidate<AccountId> {
+	/// Identifier.
+	pub who: AccountId,
+	/// Intermediary value used to sort candidates.
+	pub score: Rational128,
+	/// Sum of the stake of this candidate based on received votes.
+	approval_stake: ExtendedBalance,
+	/// Flag for being elected.
+	elected: bool,
+}
+
+/// A voter entity.
+#[derive(Clone, Default, RuntimeDebug)]
+pub struct Voter<AccountId> {
+	/// Identifier.
+	who: AccountId,
+	/// List of candidates proposed by this voter.
+	edges: Vec<Edge<AccountId>>,
+	/// The stake of this voter.
+	budget: ExtendedBalance,
+	/// Incremented each time a candidate that this voter voted for has been elected.
+	load: Rational128,
+}
+
+/// A candidate being backed by a voter.
+#[derive(Clone, Default, RuntimeDebug)]
+pub struct Edge<AccountId> {
+	/// Identifier.
+	who: AccountId,
+	/// Load of this vote.
+	load: Rational128,
+	/// Index of the candidate stored in the 'candidates' vector.
+	candidate_index: usize,
+}
+
+/// Means a particular `AccountId` was backed by `Perbill`th of a nominator's stake.
+pub type PhragmenAssignment<AccountId> = (AccountId, Perbill);
+
+/// Means a particular `AccountId` was backed by `ExtendedBalance` of a nominator's stake.
+pub type PhragmenStakedAssignment<AccountId> = (AccountId, ExtendedBalance);
+
+/// Final result of the phragmen election.
+#[derive(RuntimeDebug)]
+pub struct PhragmenResult<AccountId> {
+	/// Just winners zipped with their approval stake. Note that the approval stake is merely the
+	/// sub of their received stake and could be used for very basic sorting and approval voting.
+	pub winners: Vec<(AccountId, ExtendedBalance)>,
+	/// Individual assignments. for each tuple, the first elements is a voter and the second
+	/// is the list of candidates that it supports.
+	pub assignments: Vec<(AccountId, Vec<PhragmenAssignment<AccountId>>)>
+}
+
+/// A structure to demonstrate the phragmen result from the perspective of the candidate, i.e. how
+/// much support each candidate is receiving.
+///
+/// This complements the [`PhragmenResult`] and is needed to run the equalize post-processing.
+///
+/// This, at the current version, resembles the `Exposure` defined in the staking SRML module, yet
+/// they do not necessarily have to be the same.
+#[derive(Default, RuntimeDebug)]
+#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))]
+pub struct Support<AccountId> {
+	/// The amount of support as the effect of self-vote.
+	pub own: ExtendedBalance,
+	/// Total support.
+	pub total: ExtendedBalance,
+	/// Support from voters.
+	pub others: Vec<PhragmenStakedAssignment<AccountId>>,
+}
+
+/// A linkage from a candidate and its [`Support`].
+pub type SupportMap<A> = BTreeMap<A, Support<A>>;
+
+/// Perform election based on Phragmén algorithm.
+///
+/// Returns an `Option` the set of winners and their detailed support ratio from each voter if
+/// enough candidates are provided. Returns `None` otherwise.
+///
+/// * `candidate_count`: number of candidates to elect.
+/// * `minimum_candidate_count`: minimum number of candidates to elect. If less candidates exist,
+///   `None` is returned.
+/// * `initial_candidates`: candidates list to be elected from.
+/// * `initial_voters`: voters list.
+/// * `stake_of`: something that can return the stake stake of a particular candidate or voter.
+///
+/// This function does not strip out candidates who do not have any backing stake. It is the
+/// responsibility of the caller to make sure only those candidates who have a sensible economic
+/// value are passed in. From the perspective of this function, a candidate can easily be among the
+/// winner with no backing stake.
+pub fn elect<AccountId, Balance, FS, C>(
+	candidate_count: usize,
+	minimum_candidate_count: usize,
+	initial_candidates: Vec<AccountId>,
+	initial_voters: Vec<(AccountId, Vec<AccountId>)>,
+	stake_of: FS,
+) -> Option<PhragmenResult<AccountId>> where
+	AccountId: Default + Ord + Member,
+	Balance: Default + Copy + SimpleArithmetic,
+	for<'r> FS: Fn(&'r AccountId) -> Balance,
+	C: Convert<Balance, u64> + Convert<u128, Balance>,
+{
+	let to_votes = |b: Balance| <C as Convert<Balance, u64>>::convert(b) as ExtendedBalance;
+
+	// return structures
+	let mut elected_candidates: Vec<(AccountId, ExtendedBalance)>;
+	let mut assigned: Vec<(AccountId, Vec<PhragmenAssignment<AccountId>>)>;
+
+	// used to cache and access candidates index.
+	let mut c_idx_cache = BTreeMap::<AccountId, usize>::new();
+
+	// voters list.
+	let num_voters = initial_candidates.len() + initial_voters.len();
+	let mut voters: Vec<Voter<AccountId>> = Vec::with_capacity(num_voters);
+
+	// Iterate once to create a cache of candidates indexes. This could be optimized by being
+	// provided by the call site.
+	let mut candidates = initial_candidates
+		.into_iter()
+		.enumerate()
+		.map(|(idx, who)| {
+			c_idx_cache.insert(who.clone(), idx);
+			Candidate { who, ..Default::default() }
+		})
+		.collect::<Vec<Candidate<AccountId>>>();
+
+	// early return if we don't have enough candidates
+	if candidates.len() < minimum_candidate_count { return None; }
+
+	// collect voters. use `c_idx_cache` for fast access and aggregate `approval_stake` of
+	// candidates.
+	voters.extend(initial_voters.into_iter().map(|(who, votes)| {
+		let voter_stake = stake_of(&who);
+		let mut edges: Vec<Edge<AccountId>> = Vec::with_capacity(votes.len());
+		for v in votes {
+			if let Some(idx) = c_idx_cache.get(&v) {
+				// This candidate is valid + already cached.
+				candidates[*idx].approval_stake = candidates[*idx].approval_stake
+					.saturating_add(to_votes(voter_stake));
+				edges.push(Edge { who: v.clone(), candidate_index: *idx, ..Default::default() });
+			} // else {} would be wrong votes. We don't really care about it.
+		}
+		Voter {
+			who,
+			edges: edges,
+			budget: to_votes(voter_stake),
+			load: Rational128::zero(),
+		}
+	}));
+
+
+	// we have already checked that we have more candidates than minimum_candidate_count.
+	// run phragmen.
+	let to_elect = candidate_count.min(candidates.len());
+	elected_candidates = Vec::with_capacity(candidate_count);
+	assigned = Vec::with_capacity(candidate_count);
+
+	// main election loop
+	for _round in 0..to_elect {
+		// loop 1: initialize score
+		for c in &mut candidates {
+			if !c.elected {
+				// 1 / approval_stake == (DEN / approval_stake) / DEN. If approval_stake is zero,
+				// then the ratio should be as large as possible, essentially `infinity`.
+				if c.approval_stake.is_zero() {
+					c.score = Rational128::from_unchecked(DEN, 0);
+				} else {
+					c.score = Rational128::from(DEN / c.approval_stake, DEN);
+				}
+			}
+		}
+
+		// loop 2: increment score
+		for n in &voters {
+			for e in &n.edges {
+				let c = &mut candidates[e.candidate_index];
+				if !c.elected && !c.approval_stake.is_zero() {
+					let temp_n = multiply_by_rational(
+						n.load.n(),
+						n.budget,
+						c.approval_stake,
+					).unwrap_or(Bounded::max_value());
+					let temp_d = n.load.d();
+					let temp = Rational128::from(temp_n, temp_d);
+					c.score = c.score.lazy_saturating_add(temp);
+				}
+			}
+		}
+
+		// loop 3: find the best
+		if let Some(winner) = candidates
+			.iter_mut()
+			.filter(|c| !c.elected)
+			.min_by_key(|c| c.score)
+		{
+			// loop 3: update voter and edge load
+			winner.elected = true;
+			for n in &mut voters {
+				for e in &mut n.edges {
+					if e.who == winner.who {
+						e.load = winner.score.lazy_saturating_sub(n.load);
+						n.load = winner.score;
+					}
+				}
+			}
+
+			elected_candidates.push((winner.who.clone(), winner.approval_stake));
+		} else {
+			break
+		}
+	} // end of all rounds
+
+	// update backing stake of candidates and voters
+	for n in &mut voters {
+		let mut assignment = (n.who.clone(), vec![]);
+		for e in &mut n.edges {
+			if elected_candidates.iter().position(|(ref c, _)| *c == e.who).is_some() {
+				let per_bill_parts =
+				{
+					if n.load == e.load {
+						// Full support. No need to calculate.
+						Perbill::accuracy().into()
+					} else {
+						if e.load.d() == n.load.d() {
+							// return e.load / n.load.
+							let desired_scale: u128 = Perbill::accuracy().into();
+							multiply_by_rational(
+								desired_scale,
+								e.load.n(),
+								n.load.n(),
+							).unwrap_or(Bounded::max_value())
+						} else {
+							// defensive only. Both edge and nominator loads are built from
+							// scores, hence MUST have the same denominator.
+							Zero::zero()
+						}
+					}
+				};
+				// safer to .min() inside as well to argue as u32 is safe.
+				let per_thing = Perbill::from_parts(
+					per_bill_parts.min(Perbill::accuracy().into()) as u32
+				);
+				assignment.1.push((e.who.clone(), per_thing));
+			}
+		}
+
+		if assignment.1.len() > 0 {
+			// To ensure an assertion indicating: no stake from the nominator going to waste,
+			// we add a minimal post-processing to equally assign all of the leftover stake ratios.
+			let vote_count = assignment.1.len() as u32;
+			let len = assignment.1.len();
+			let sum = assignment.1.iter()
+				.map(|a| a.1.deconstruct())
+				.sum::<u32>();
+			let accuracy = Perbill::accuracy();
+			let diff = accuracy.checked_sub(sum).unwrap_or(0);
+			let diff_per_vote = (diff / vote_count).min(accuracy);
+
+			if diff_per_vote > 0 {
+				for i in 0..len {
+					let current_ratio = assignment.1[i % len].1;
+					let next_ratio = current_ratio
+						.saturating_add(Perbill::from_parts(diff_per_vote));
+					assignment.1[i % len].1 = next_ratio;
+				}
+			}
+
+			// `remainder` is set to be less than maximum votes of a nominator (currently 16).
+			// safe to cast it to usize.
+			let remainder = diff - diff_per_vote * vote_count;
+			for i in 0..remainder as usize {
+				let current_ratio = assignment.1[i % len].1;
+				let next_ratio = current_ratio.saturating_add(Perbill::from_parts(1));
+				assignment.1[i % len].1 = next_ratio;
+			}
+			assigned.push(assignment);
+		}
+	}
+
+	Some(PhragmenResult {
+		winners: elected_candidates,
+		assignments: assigned,
+	})
+}
+
+/// Build the support map from the given phragmen result.
+pub fn build_support_map<Balance, AccountId, FS, C>(
+	elected_stashes: &Vec<AccountId>,
+	assignments: &Vec<(AccountId, Vec<PhragmenAssignment<AccountId>>)>,
+	stake_of: FS,
+) -> SupportMap<AccountId> where
+	AccountId: Default + Ord + Member,
+	Balance: Default + Copy + SimpleArithmetic,
+	C: Convert<Balance, u64> + Convert<u128, Balance>,
+	for<'r> FS: Fn(&'r AccountId) -> Balance,
+{
+	let to_votes = |b: Balance| <C as Convert<Balance, u64>>::convert(b) as ExtendedBalance;
+	// Initialize the support of each candidate.
+	let mut supports = <SupportMap<AccountId>>::new();
+	elected_stashes
+		.iter()
+		.for_each(|e| { supports.insert(e.clone(), Default::default()); });
+
+	// build support struct.
+	for (n, assignment) in assignments.iter() {
+		for (c, per_thing) in assignment.iter() {
+			let nominator_stake = to_votes(stake_of(n));
+			// AUDIT: it is crucially important for the `Mul` implementation of all
+			// per-things to be sound.
+			let other_stake = *per_thing * nominator_stake;
+			if let Some(support) = supports.get_mut(c) {
+				if c == n {
+					// This is a nomination from `n` to themselves. This will increase both the
+					// `own` and `total` field.
+					debug_assert!(*per_thing == Perbill::one()); // TODO: deal with this: do we want it?
+					support.own = support.own.saturating_add(other_stake);
+					support.total = support.total.saturating_add(other_stake);
+				} else {
+					// This is a nomination from `n` to someone else. Increase `total` and add an entry
+					// inside `others`.
+					// For an astronomically rich validator with more astronomically rich
+					// set of nominators, this might saturate.
+					support.total = support.total.saturating_add(other_stake);
+					support.others.push((n.clone(), other_stake));
+				}
+			}
+		}
+	}
+	supports
+}
+
+/// Performs equalize post-processing to the output of the election algorithm. This happens in
+/// rounds. The number of rounds and the maximum diff-per-round tolerance can be tuned through input
+/// parameters.
+///
+/// No value is returned from the function and the `supports` parameter is updated.
+///
+/// * `assignments`: exactly the same is the output of phragmen.
+/// * `supports`: mutable reference to s `SupportMap`. This parameter is updated.
+/// * `tolerance`: maximum difference that can occur before an early quite happens.
+/// * `iterations`: maximum number of iterations that will be processed.
+/// * `stake_of`: something that can return the stake stake of a particular candidate or voter.
+pub fn equalize<Balance, AccountId, C, FS>(
+	mut assignments: Vec<(AccountId, Vec<PhragmenStakedAssignment<AccountId>>)>,
+	supports: &mut SupportMap<AccountId>,
+	tolerance: ExtendedBalance,
+	iterations: usize,
+	stake_of: FS,
+) where
+	C: Convert<Balance, u64> + Convert<u128, Balance>,
+	for<'r> FS: Fn(&'r AccountId) -> Balance,
+	AccountId: Ord + Clone,
+{
+	// prepare the data for equalise
+	for _i in 0..iterations {
+		let mut max_diff = 0;
+
+		for (voter, assignment) in assignments.iter_mut() {
+			let voter_budget = stake_of(&voter);
+
+			let diff = do_equalize::<_, _, C>(
+				voter,
+				voter_budget,
+				assignment,
+				supports,
+				tolerance,
+			);
+			if diff > max_diff { max_diff = diff; }
+		}
+
+		if max_diff < tolerance {
+			break;
+		}
+	}
+}
+
+/// actually perform equalize. same interface is `equalize`. Just called in loops with a check for
+/// maximum difference.
+fn do_equalize<Balance, AccountId, C>(
+	voter: &AccountId,
+	budget_balance: Balance,
+	elected_edges: &mut Vec<PhragmenStakedAssignment<AccountId>>,
+	support_map: &mut SupportMap<AccountId>,
+	tolerance: ExtendedBalance
+) -> ExtendedBalance where
+	C: Convert<Balance, u64> + Convert<u128, Balance>,
+	AccountId: Ord + Clone,
+{
+	let to_votes = |b: Balance|
+		<C as Convert<Balance, u64>>::convert(b) as ExtendedBalance;
+	let budget = to_votes(budget_balance);
+
+	// Nothing to do. This voter had nothing useful.
+	// Defensive only. Assignment list should always be populated.
+	if elected_edges.is_empty() { return 0; }
+
+	let stake_used = elected_edges
+		.iter()
+		.fold(0 as ExtendedBalance, |s, e| s.saturating_add(e.1));
+
+	let backed_stakes_iter = elected_edges
+		.iter()
+		.filter_map(|e| support_map.get(&e.0))
+		.map(|e| e.total);
+
+	let backing_backed_stake = elected_edges
+		.iter()
+		.filter(|e| e.1 > 0)
+		.filter_map(|e| support_map.get(&e.0))
+		.map(|e| e.total)
+		.collect::<Vec<ExtendedBalance>>();
+
+	let mut difference;
+	if backing_backed_stake.len() > 0 {
+		let max_stake = backing_backed_stake
+			.iter()
+			.max()
+			.expect("vector with positive length will have a max; qed");
+		let min_stake = backed_stakes_iter
+			.min()
+			.expect("iterator with positive length will have a min; qed");
+
+		difference = max_stake.saturating_sub(min_stake);
+		difference = difference.saturating_add(budget.saturating_sub(stake_used));
+		if difference < tolerance {
+			return difference;
+		}
+	} else {
+		difference = budget;
+	}
+
+	// Undo updates to support
+	elected_edges.iter_mut().for_each(|e| {
+		if let Some(support) = support_map.get_mut(&e.0) {
+			support.total = support.total.saturating_sub(e.1);
+			support.others.retain(|i_support| i_support.0 != *voter);
+		}
+		e.1 = 0;
+	});
+
+	elected_edges.sort_unstable_by_key(|e|
+		if let Some(e) = support_map.get(&e.0) { e.total } else { Zero::zero() }
+	);
+
+	let mut cumulative_stake: ExtendedBalance = 0;
+	let mut last_index = elected_edges.len() - 1;
+	let mut idx = 0usize;
+	for e in &mut elected_edges[..] {
+		if let Some(support) = support_map.get_mut(&e.0) {
+			let stake = support.total;
+			let stake_mul = stake.saturating_mul(idx as ExtendedBalance);
+			let stake_sub = stake_mul.saturating_sub(cumulative_stake);
+			if stake_sub > budget {
+				last_index = idx.checked_sub(1).unwrap_or(0);
+				break;
+			}
+			cumulative_stake = cumulative_stake.saturating_add(stake);
+		}
+		idx += 1;
+	}
+
+	let last_stake = elected_edges[last_index].1;
+	let split_ways = last_index + 1;
+	let excess = budget
+		.saturating_add(cumulative_stake)
+		.saturating_sub(last_stake.saturating_mul(split_ways as ExtendedBalance));
+	elected_edges.iter_mut().take(split_ways).for_each(|e| {
+		if let Some(support) = support_map.get_mut(&e.0) {
+			e.1 = (excess / split_ways as ExtendedBalance)
+				.saturating_add(last_stake)
+				.saturating_sub(support.total);
+			support.total = support.total.saturating_add(e.1);
+			support.others.push((voter.clone(), e.1));
+		}
+	});
+
+	difference
+}

substrate/primitives/phragmen/src/mock.rs

@@ -0,0 +1,412 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Mock file for phragmen.
+
+#![cfg(test)]
+
+use crate::{elect, PhragmenResult, PhragmenAssignment};
+use sr_primitives::{
+	assert_eq_error_rate, Perbill,
+	traits::{Convert, Member, SaturatedConversion}
+};
+use rstd::collections::btree_map::BTreeMap;
+
+pub(crate) struct TestCurrencyToVote;
+impl Convert<Balance, u64> for TestCurrencyToVote {
+	fn convert(x: Balance) -> u64 { x.saturated_into() }
+}
+impl Convert<u128, Balance> for TestCurrencyToVote {
+	fn convert(x: u128) -> Balance { x }
+}
+
+#[derive(Default, Debug)]
+pub(crate) struct _Candidate<A> {
+	who: A,
+	score: f64,
+	approval_stake: f64,
+	elected: bool,
+}
+
+#[derive(Default, Debug)]
+pub(crate) struct _Voter<A> {
+	who: A,
+	edges: Vec<_Edge<A>>,
+	budget: f64,
+	load: f64,
+}
+
+#[derive(Default, Debug)]
+pub(crate) struct _Edge<A> {
+	who: A,
+	load: f64,
+	candidate_index: usize,
+}
+
+#[derive(Default, Debug, PartialEq)]
+pub(crate) struct _Support<A> {
+	pub own: f64,
+	pub total: f64,
+	pub others: Vec<_PhragmenAssignment<A>>,
+}
+
+pub(crate) type _PhragmenAssignment<A> = (A, f64);
+pub(crate) type _SupportMap<A> = BTreeMap<A, _Support<A>>;
+
+pub(crate) type Balance = u128;
+pub(crate) type AccountId = u64;
+
+#[derive(Debug, Clone)]
+pub(crate) struct _PhragmenResult<A: Clone> {
+	pub winners: Vec<(A, Balance)>,
+	pub assignments: Vec<(A, Vec<_PhragmenAssignment<A>>)>
+}
+
+pub(crate) fn auto_generate_self_voters<A: Clone>(candidates: &[A]) -> Vec<(A, Vec<A>)> {
+	candidates.iter().map(|c| (c.clone(), vec![c.clone()])).collect()
+}
+
+pub(crate) fn elect_float<A, FS>(
+	candidate_count: usize,
+	minimum_candidate_count: usize,
+	initial_candidates: Vec<A>,
+	initial_voters: Vec<(A, Vec<A>)>,
+	stake_of: FS,
+) -> Option<_PhragmenResult<A>> where
+	A: Default + Ord + Member + Copy,
+	for<'r> FS: Fn(&'r A) -> Balance,
+{
+	let mut elected_candidates: Vec<(A, Balance)>;
+	let mut assigned: Vec<(A, Vec<_PhragmenAssignment<A>>)>;
+	let mut c_idx_cache = BTreeMap::<A, usize>::new();
+	let num_voters = initial_candidates.len() + initial_voters.len();
+	let mut voters: Vec<_Voter<A>> = Vec::with_capacity(num_voters);
+
+	let mut candidates = initial_candidates
+		.into_iter()
+		.enumerate()
+		.map(|(idx, who)| {
+			c_idx_cache.insert(who.clone(), idx);
+			_Candidate { who, ..Default::default() }
+		})
+		.collect::<Vec<_Candidate<A>>>();
+
+	if candidates.len() < minimum_candidate_count {
+		return None;
+	}
+
+	voters.extend(initial_voters.into_iter().map(|(who, votes)| {
+		let voter_stake = stake_of(&who) as f64;
+		let mut edges: Vec<_Edge<A>> = Vec::with_capacity(votes.len());
+		for v in votes {
+			if let Some(idx) = c_idx_cache.get(&v) {
+				candidates[*idx].approval_stake = candidates[*idx].approval_stake + voter_stake;
+				edges.push(
+					_Edge { who: v.clone(), candidate_index: *idx, ..Default::default() }
+				);
+			}
+		}
+		_Voter {
+			who,
+			edges: edges,
+			budget: voter_stake,
+			load: 0f64,
+		}
+	}));
+
+	let to_elect = candidate_count.min(candidates.len());
+	elected_candidates = Vec::with_capacity(candidate_count);
+	assigned = Vec::with_capacity(candidate_count);
+
+	for _round in 0..to_elect {
+		for c in &mut candidates {
+			if !c.elected {
+				c.score = 1.0 / c.approval_stake;
+			}
+		}
+		for n in &voters {
+			for e in &n.edges {
+				let c = &mut candidates[e.candidate_index];
+				if !c.elected && !(c.approval_stake == 0f64) {
+					c.score += n.budget * n.load / c.approval_stake;
+				}
+			}
+		}
+
+		if let Some(winner) = candidates
+			.iter_mut()
+			.filter(|c| !c.elected)
+			.min_by(|x, y| x.score.partial_cmp(&y.score).unwrap_or(rstd::cmp::Ordering::Equal))
+		{
+			winner.elected = true;
+			for n in &mut voters {
+				for e in &mut n.edges {
+					if e.who == winner.who {
+						e.load = winner.score - n.load;
+						n.load = winner.score;
+					}
+				}
+			}
+
+			elected_candidates.push((winner.who.clone(), winner.approval_stake as Balance));
+		} else {
+			break
+		}
+	}
+
+	for n in &mut voters {
+		let mut assignment = (n.who.clone(), vec![]);
+		for e in &mut n.edges {
+			if let Some(c) = elected_candidates.iter().cloned().map(|(c, _)| c).find(|c| *c == e.who) {
+				if c != n.who {
+					let ratio = e.load / n.load;
+					assignment.1.push((e.who.clone(), ratio));
+				}
+			}
+		}
+		if assignment.1.len() > 0 {
+			assigned.push(assignment);
+		}
+	}
+
+	Some(_PhragmenResult {
+		winners: elected_candidates,
+		assignments: assigned,
+	})
+}
+
+pub(crate) fn equalize_float<A, FS>(
+	mut assignments: Vec<(A, Vec<_PhragmenAssignment<A>>)>,
+	supports: &mut _SupportMap<A>,
+	tolerance: f64,
+	iterations: usize,
+	stake_of: FS,
+) where
+	for<'r> FS: Fn(&'r A) -> Balance,
+	A: Ord + Clone + std::fmt::Debug,
+{
+	for _i in 0..iterations {
+		let mut max_diff = 0.0;
+		for (voter, assignment) in assignments.iter_mut() {
+			let voter_budget = stake_of(&voter);
+			let diff = do_equalize_float(
+				voter,
+				voter_budget,
+				assignment,
+				supports,
+				tolerance,
+			);
+			if diff > max_diff { max_diff = diff; }
+		}
+
+		if max_diff < tolerance {
+			break;
+		}
+	}
+}
+
+pub(crate) fn do_equalize_float<A>(
+	voter: &A,
+	budget_balance: Balance,
+	elected_edges: &mut Vec<_PhragmenAssignment<A>>,
+	support_map: &mut _SupportMap<A>,
+	tolerance: f64
+) -> f64 where
+	A: Ord + Clone,
+{
+	let budget = budget_balance as f64;
+	if elected_edges.is_empty() { return 0.0; }
+
+	let stake_used = elected_edges
+		.iter()
+		.fold(0.0, |s, e| s + e.1);
+
+	let backed_stakes_iter = elected_edges
+		.iter()
+		.filter_map(|e| support_map.get(&e.0))
+		.map(|e| e.total);
+
+	let backing_backed_stake = elected_edges
+		.iter()
+		.filter(|e| e.1 > 0.0)
+		.filter_map(|e| support_map.get(&e.0))
+		.map(|e| e.total)
+		.collect::<Vec<f64>>();
+
+	let mut difference;
+	if backing_backed_stake.len() > 0 {
+		let max_stake = backing_backed_stake
+			.iter()
+			.max_by(|x, y| x.partial_cmp(&y).unwrap_or(rstd::cmp::Ordering::Equal))
+			.expect("vector with positive length will have a max; qed");
+		let min_stake = backed_stakes_iter
+			.min_by(|x, y| x.partial_cmp(&y).unwrap_or(rstd::cmp::Ordering::Equal))
+			.expect("iterator with positive length will have a min; qed");
+
+		difference = max_stake - min_stake;
+		difference = difference + budget - stake_used;
+		if difference < tolerance {
+			return difference;
+		}
+	} else {
+		difference = budget;
+	}
+
+	// Undo updates to support
+	elected_edges.iter_mut().for_each(|e| {
+		if let Some(support) = support_map.get_mut(&e.0) {
+			support.total = support.total - e.1;
+			support.others.retain(|i_support| i_support.0 != *voter);
+		}
+		e.1 = 0.0;
+	});
+
+	// todo: rewrite.
+	elected_edges.sort_unstable_by(|x, y|
+		if let Some(x) = support_map.get(&x.0) {
+			if let Some(y) = support_map.get(&y.0) {
+				x.total.partial_cmp(&y.total).unwrap_or(rstd::cmp::Ordering::Equal)
+			} else {
+				rstd::cmp::Ordering::Equal
+			}
+		} else {
+			rstd::cmp::Ordering::Equal
+		}
+	);
+
+	let mut cumulative_stake = 0.0;
+	let mut last_index = elected_edges.len() - 1;
+	elected_edges.iter_mut().enumerate().for_each(|(idx, e)| {
+		if let Some(support) = support_map.get_mut(&e.0) {
+			let stake = support.total;
+			let stake_mul = stake * (idx as f64);
+			let stake_sub = stake_mul - cumulative_stake;
+			if stake_sub > budget {
+				last_index = idx.checked_sub(1).unwrap_or(0);
+				return
+			}
+			cumulative_stake = cumulative_stake + stake;
+		}
+	});
+
+	let last_stake = elected_edges[last_index].1;
+	let split_ways = last_index + 1;
+	let excess = budget + cumulative_stake - last_stake * (split_ways as f64);
+	elected_edges.iter_mut().take(split_ways).for_each(|e| {
+		if let Some(support) = support_map.get_mut(&e.0) {
+			e.1 = excess / (split_ways as f64) + last_stake - support.total;
+			support.total = support.total + e.1;
+			support.others.push((voter.clone(), e.1));
+		}
+	});
+
+	difference
+}
+
+
+pub(crate) fn create_stake_of(stakes: &[(AccountId, Balance)])
+	-> Box<dyn Fn(&AccountId) -> Balance>
+{
+	let mut storage = BTreeMap::<AccountId, Balance>::new();
+	stakes.iter().for_each(|s| { storage.insert(s.0, s.1); });
+	let stake_of = move |who: &AccountId| -> Balance { storage.get(who).unwrap().to_owned() };
+	Box::new(stake_of)
+}
+
+
+pub fn check_assignments(assignments: Vec<(AccountId, Vec<PhragmenAssignment<AccountId>>)>) {
+	for (_, a) in assignments {
+		let sum: u32 = a.iter().map(|(_, p)| p.deconstruct()).sum();
+		assert_eq_error_rate!(sum, Perbill::accuracy(), 5);
+	}
+}
+
+pub(crate) fn run_and_compare(
+	candidates: Vec<AccountId>,
+	voters: Vec<(AccountId, Vec<AccountId>)>,
+	stake_of: Box<dyn Fn(&AccountId) -> Balance>,
+	to_elect: usize,
+	min_to_elect: usize,
+) {
+	// run fixed point code.
+	let PhragmenResult { winners, assignments } = elect::<_, _, _, TestCurrencyToVote>(
+		to_elect,
+		min_to_elect,
+		candidates.clone(),
+		voters.clone(),
+		&stake_of,
+	).unwrap();
+
+	// run float poc code.
+	let truth_value = elect_float(
+		to_elect,
+		min_to_elect,
+		candidates,
+		voters,
+		&stake_of,
+	).unwrap();
+
+	assert_eq!(winners, truth_value.winners);
+
+	for (nominator, assigned) in assignments.clone() {
+		if let Some(float_assignments) = truth_value.assignments.iter().find(|x| x.0 == nominator) {
+			for (candidate, per_thingy) in assigned {
+				if let Some(float_assignment) = float_assignments.1.iter().find(|x| x.0 == candidate ) {
+					assert_eq_error_rate!(
+						Perbill::from_fraction(float_assignment.1).deconstruct(),
+						per_thingy.deconstruct(),
+						1,
+					);
+				} else {
+					panic!("candidate mismatch. This should never happen.")
+				}
+			}
+		} else {
+			panic!("nominator mismatch. This should never happen.")
+		}
+	}
+
+	check_assignments(assignments);
+}
+
+pub(crate) fn build_support_map<FS>(
+	result: &mut _PhragmenResult<AccountId>,
+	stake_of: FS,
+) -> _SupportMap<AccountId>
+	where for<'r> FS: Fn(&'r AccountId) -> Balance
+{
+	let mut supports = <_SupportMap<AccountId>>::new();
+	result.winners
+		.iter()
+		.map(|(e, _)| (e, stake_of(e) as f64))
+		.for_each(|(e, s)| {
+			let item = _Support { own: s, total: s, ..Default::default() };
+			supports.insert(e.clone(), item);
+		});
+
+	for (n, assignment) in result.assignments.iter_mut() {
+		for (c, r) in assignment.iter_mut() {
+			let nominator_stake = stake_of(n) as f64;
+			let other_stake = nominator_stake * *r;
+			if let Some(support) = supports.get_mut(c) {
+				support.total = support.total + other_stake;
+				support.others.push((n.clone(), other_stake));
+			}
+			*r = other_stake;
+		}
+	}
+	supports
+}

substrate/primitives/phragmen/src/tests.rs

@@ -0,0 +1,356 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Tests for phragmen.
+
+#![cfg(test)]
+
+use crate::mock::*;
+use crate::{elect, PhragmenResult};
+use support::assert_eq_uvec;
+use sr_primitives::Perbill;
+
+#[test]
+fn float_phragmen_poc_works() {
+	let candidates = vec![1, 2, 3];
+	let voters = vec![
+		(10, vec![1, 2]),
+		(20, vec![1, 3]),
+		(30, vec![2, 3]),
+	];
+	let stake_of = create_stake_of(&[(10, 10), (20, 20), (30, 30), (1, 0), (2, 0), (3, 0)]);
+	let mut phragmen_result = elect_float(2, 2, candidates, voters, &stake_of).unwrap();
+	let winners = phragmen_result.clone().winners;
+	let assignments = phragmen_result.clone().assignments;
+
+	assert_eq_uvec!(winners, vec![(2, 40), (3, 50)]);
+	assert_eq_uvec!(
+		assignments,
+		vec![
+			(10, vec![(2, 1.0)]),
+			(20, vec![(3, 1.0)]),
+			(30, vec![(2, 0.5), (3, 0.5)]),
+		]
+	);
+
+	let mut support_map = build_support_map(&mut phragmen_result, &stake_of);
+
+	assert_eq!(
+		support_map.get(&2).unwrap(),
+		&_Support { own: 0.0, total: 25.0, others: vec![(10u64, 10.0), (30u64, 15.0)]}
+	);
+	assert_eq!(
+		support_map.get(&3).unwrap(),
+		&_Support { own: 0.0, total: 35.0, others: vec![(20u64, 20.0), (30u64, 15.0)]}
+	);
+
+	equalize_float(phragmen_result.assignments, &mut support_map, 0.0, 2, stake_of);
+
+	assert_eq!(
+		support_map.get(&2).unwrap(),
+		&_Support { own: 0.0, total: 30.0, others: vec![(10u64, 10.0), (30u64, 20.0)]}
+	);
+	assert_eq!(
+		support_map.get(&3).unwrap(),
+		&_Support { own: 0.0, total: 30.0, others: vec![(20u64, 20.0), (30u64, 10.0)]}
+	);
+}
+
+#[test]
+fn phragmen_poc_works() {
+	let candidates = vec![1, 2, 3];
+	let voters = vec![
+		(10, vec![1, 2]),
+		(20, vec![1, 3]),
+		(30, vec![2, 3]),
+	];
+
+	let PhragmenResult { winners, assignments } = elect::<_, _, _, TestCurrencyToVote>(
+		2,
+		2,
+		candidates,
+		voters,
+		create_stake_of(&[(10, 10), (20, 20), (30, 30)]),
+	).unwrap();
+
+	assert_eq_uvec!(winners, vec![(2, 40), (3, 50)]);
+	assert_eq_uvec!(
+		assignments,
+		vec![
+			(10, vec![(2, Perbill::from_percent(100))]),
+			(20, vec![(3, Perbill::from_percent(100))]),
+			(30, vec![(2, Perbill::from_percent(100/2)), (3, Perbill::from_percent(100/2))]),
+		]
+	);
+}
+
+#[test]
+fn phragmen_poc_2_works() {
+	let candidates = vec![10, 20, 30];
+	let voters = vec![
+		(2, vec![10, 20, 30]),
+		(4, vec![10, 20, 40]),
+	];
+	let stake_of = create_stake_of(&[
+		(10, 1000),
+		(20, 1000),
+		(30, 1000),
+		(40, 1000),
+		(2, 500),
+		(4, 500),
+	]);
+
+	run_and_compare(candidates, voters, stake_of, 2, 2);
+}
+
+#[test]
+fn phragmen_poc_3_works() {
+	let candidates = vec![10, 20, 30];
+	let voters = vec![
+		(2, vec![10, 20, 30]),
+		(4, vec![10, 20, 40]),
+	];
+	let stake_of = create_stake_of(&[
+		(10, 1000),
+		(20, 1000),
+		(30, 1000),
+		(2, 50),
+		(4, 1000),
+	]);
+
+	run_and_compare(candidates, voters, stake_of, 2, 2);
+}
+
+#[test]
+fn phragmen_accuracy_on_large_scale_only_validators() {
+	// because of this particular situation we had per_u128 and now rational128. In practice, a
+	// candidate can have the maximum amount of tokens, and also supported by the maximum.
+	let candidates = vec![1, 2, 3, 4, 5];
+	let stake_of = create_stake_of(&[
+		(1, (u64::max_value() - 1).into()),
+		(2, (u64::max_value() - 4).into()),
+		(3, (u64::max_value() - 5).into()),
+		(4, (u64::max_value() - 3).into()),
+		(5, (u64::max_value() - 2).into()),
+	]);
+
+	let PhragmenResult { winners, assignments } = elect::<_, _, _, TestCurrencyToVote>(
+		2,
+		2,
+		candidates.clone(),
+		auto_generate_self_voters(&candidates),
+		stake_of,
+	).unwrap();
+
+	assert_eq_uvec!(winners, vec![(1, 18446744073709551614u128), (5, 18446744073709551613u128)]);
+	assert_eq!(assignments.len(), 2);
+	check_assignments(assignments);
+}
+
+#[test]
+fn phragmen_accuracy_on_large_scale_validators_and_nominators() {
+	let candidates = vec![1, 2, 3, 4, 5];
+	let mut voters = vec![
+		(13, vec![1, 3, 5]),
+		(14, vec![2, 4]),
+	];
+	voters.extend(auto_generate_self_voters(&candidates));
+	let stake_of = create_stake_of(&[
+		(1,  (u64::max_value() - 1).into()),
+		(2,  (u64::max_value() - 4).into()),
+		(3,  (u64::max_value() - 5).into()),
+		(4,  (u64::max_value() - 3).into()),
+		(5,  (u64::max_value() - 2).into()),
+		(13, (u64::max_value() - 10).into()),
+		(14, u64::max_value().into()),
+	]);
+
+	let PhragmenResult { winners, assignments } = elect::<_, _, _, TestCurrencyToVote>(
+		2,
+		2,
+		candidates,
+		voters,
+		stake_of,
+	).unwrap();
+
+	assert_eq_uvec!(winners, vec![(2, 36893488147419103226u128), (1, 36893488147419103219u128)]);
+	assert_eq!(
+		assignments,
+		vec![
+			(13, vec![(1, Perbill::one())]),
+			(14, vec![(2, Perbill::one())]),
+			(1, vec![(1, Perbill::one())]),
+			(2, vec![(2, Perbill::one())]),
+		]
+	);
+	check_assignments(assignments);
+}
+
+#[test]
+fn phragmen_accuracy_on_small_scale_self_vote() {
+	let candidates = vec![40, 10, 20, 30];
+	let voters = auto_generate_self_voters(&candidates);
+	let stake_of = create_stake_of(&[
+		(40, 0),
+		(10, 1),
+		(20, 2),
+		(30, 1),
+	]);
+
+	let PhragmenResult { winners, assignments: _ } = elect::<_, _, _, TestCurrencyToVote>(
+		3,
+		3,
+		candidates,
+		voters,
+		stake_of,
+	).unwrap();
+
+	assert_eq_uvec!(winners, vec![(20, 2), (10, 1), (30, 1)]);
+}
+
+#[test]
+fn phragmen_accuracy_on_small_scale_no_self_vote() {
+	let candidates = vec![40, 10, 20, 30];
+	let voters = vec![
+		(1, vec![10]),
+		(2, vec![20]),
+		(3, vec![30]),
+		(4, vec![40]),
+	];
+	let stake_of = create_stake_of(&[
+		(40, 1000), // don't care
+		(10, 1000), // don't care
+		(20, 1000), // don't care
+		(30, 1000), // don't care
+		(4, 0),
+		(1, 1),
+		(2, 2),
+		(3, 1),
+	]);
+
+	let PhragmenResult { winners, assignments: _ } = elect::<_, _, _, TestCurrencyToVote>(
+		3,
+		3,
+		candidates,
+		voters,
+		stake_of,
+	).unwrap();
+
+	assert_eq_uvec!(winners, vec![(20, 2), (10, 1), (30, 1)]);
+}
+
+#[test]
+fn phragmen_large_scale_test() {
+	let candidates = vec![2, 4, 6, 8, 10, 12, 14, 16 ,18, 20, 22, 24];
+	let mut voters = vec![
+		(50, vec![2, 4, 6, 8, 10, 12, 14, 16 ,18, 20, 22, 24]),
+	];
+	voters.extend(auto_generate_self_voters(&candidates));
+	let stake_of = create_stake_of(&[
+		(2,  1),
+		(4,  100),
+		(6,  1000000),
+		(8,  100000000001000),
+		(10, 100000000002000),
+		(12, 100000000003000),
+		(14, 400000000000000),
+		(16, 400000000001000),
+		(18, 18000000000000000),
+		(20, 20000000000000000),
+		(22, 500000000000100000),
+		(24, 500000000000200000),
+		(50, 990000000000000000),
+	]);
+
+	let PhragmenResult { winners, assignments } = elect::<_, _, _, TestCurrencyToVote>(
+		2,
+		2,
+		candidates,
+		voters,
+		stake_of,
+	).unwrap();
+
+	assert_eq_uvec!(winners, vec![(24, 1490000000000200000u128), (22, 1490000000000100000u128)]);
+	check_assignments(assignments);
+}
+
+#[test]
+fn phragmen_large_scale_test_2() {
+	let nom_budget: u64 = 1_000_000_000_000_000_000;
+	let c_budget: u64 = 4_000_000;
+
+	let candidates = vec![2, 4];
+	let mut voters = vec![(50, vec![2, 4])];
+	voters.extend(auto_generate_self_voters(&candidates));
+
+	let stake_of = create_stake_of(&[
+		(2,  c_budget.into()),
+		(4,  c_budget.into()),
+		(50, nom_budget.into()),
+	]);
+
+	let PhragmenResult { winners, assignments } = elect::<_, _, _, TestCurrencyToVote>(
+		2,
+		2,
+		candidates,
+		voters,
+		stake_of,
+	).unwrap();
+
+	assert_eq_uvec!(winners, vec![(2, 1000000000004000000u128), (4, 1000000000004000000u128)]);
+	assert_eq!(
+		assignments,
+		vec![
+			(50, vec![(2, Perbill::from_parts(500000001)), (4, Perbill::from_parts(499999999))]),
+			(2, vec![(2, Perbill::one())]),
+			(4, vec![(4, Perbill::one())]),
+		],
+	);
+	check_assignments(assignments);
+}
+
+#[test]
+fn phragmen_linear_equalize() {
+	let candidates = vec![11, 21, 31, 41, 51, 61, 71];
+	let voters = vec![
+		(2, vec![11]),
+		(4, vec![11, 21]),
+		(6, vec![21, 31]),
+		(8, vec![31, 41]),
+		(110, vec![41, 51]),
+		(120, vec![51, 61]),
+		(130, vec![61, 71]),
+	];
+	let stake_of = create_stake_of(&[
+		(11, 1000),
+		(21, 1000),
+		(31, 1000),
+		(41, 1000),
+		(51, 1000),
+		(61, 1000),
+		(71, 1000),
+
+		(2, 2000),
+		(4, 1000),
+		(6, 1000),
+		(8, 1000),
+		(110, 1000),
+		(120, 1000),
+		(130, 1000),
+	]);
+
+	run_and_compare(candidates, voters, stake_of, 2, 2);
+}

substrate/primitives/rpc/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "substrate-rpc-primitives"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+serde = { version = "1.0.101", features = ["derive"] }
+primitives = { package = "substrate-primitives", path = "../core" }

substrate/primitives/rpc/src/lib.rs

@@ -0,0 +1,21 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Substrate RPC primitives and utilities.
+
+#![warn(missing_docs)]
+
+pub mod number;

substrate/primitives/rpc/src/number.rs

@@ -0,0 +1,74 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Chain RPC Block number type.
+
+use serde::{Serialize, Deserialize};
+use std::{convert::TryFrom, fmt::Debug};
+use primitives::U256;
+
+/// RPC Block number type
+///
+/// We allow two representations of the block number as input.
+/// Either we deserialize to the type that is specified in the block type
+/// or we attempt to parse given hex value.
+/// We do that for consistency with the returned type, default generic header
+/// serializes block number as hex to avoid overflows in JavaScript.
+#[derive(Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum NumberOrHex<Number> {
+	/// The original header number type of block.
+	Number(Number),
+	/// Hex representation of the block number.
+	Hex(U256),
+}
+
+impl<Number: TryFrom<u64> + From<u32> + Debug + PartialOrd> NumberOrHex<Number> {
+	/// Attempts to convert into concrete block number.
+	///
+	/// Fails in case hex number is too big.
+	pub fn to_number(self) -> Result<Number, String> {
+		let num = match self {
+			NumberOrHex::Number(n) => n,
+			NumberOrHex::Hex(h) => {
+				let l = h.low_u64();
+				if U256::from(l) != h {
+					return Err(format!("`{}` does not fit into u64 type; unsupported for now.", h))
+				} else {
+					Number::try_from(l)
+						.map_err(|_| format!("`{}` does not fit into block number type.", h))?
+				}
+			},
+		};
+		// FIXME <2329>: Database seems to limit the block number to u32 for no reason
+		if num > Number::from(u32::max_value()) {
+			return Err(format!("`{:?}` > u32::max_value(), the max block number is u32.", num))
+		}
+		Ok(num)
+	}
+}
+
+impl From<u64> for NumberOrHex<u64> {
+	fn from(n: u64) -> Self {
+		NumberOrHex::Number(n)
+	}
+}
+
+impl<Number> From<U256> for NumberOrHex<Number> {
+	fn from(n: U256) -> Self {
+		NumberOrHex::Hex(n)
+	}
+}

substrate/primitives/runtime-interface/Cargo.toml

@@ -0,0 +1,41 @@
+[package]
+name = "substrate-runtime-interface"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+wasm-interface = { package = "substrate-wasm-interface", path = "../wasm-interface", optional = true }
+rstd = { package = "sr-std", path = "../sr-std", default-features = false }
+substrate-runtime-interface-proc-macro = { path = "proc-macro" }
+externalities = { package = "substrate-externalities", path = "../externalities", optional = true }
+codec = { package = "parity-scale-codec", version = "1.0.6", default-features = false }
+environmental = { version = "1.0.2", optional = true }
+static_assertions = "1.0.0"
+primitive-types = { version = "0.6.1", default-features = false }
+
+[dev-dependencies]
+executor = { package = "substrate-executor", path = "../../client/executor" }
+test-wasm = { package = "substrate-runtime-interface-test-wasm", path = "test-wasm" }
+state_machine = { package = "substrate-state-machine", path = "../../primitives/state-machine"  }
+primitives = { package = "substrate-primitives", path = "../core" }
+runtime-io = { package = "sr-io", path = "../sr-io" }
+
+[features]
+default = [ "std" ]
+std = [
+	"wasm-interface",
+	"rstd/std",
+	"codec/std",
+	"externalities",
+	"environmental",
+	"primitive-types/std",
+]
+
+# ATTENTION
+#
+# Only use when you know what you are doing.
+#
+# Disables static assertions in `impls.rs` that checks the word size. To prevent any footgun, the
+# check is changed into a runtime check.
+disable_target_static_assertions = []

substrate/primitives/runtime-interface/proc-macro/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "substrate-runtime-interface-proc-macro"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+syn = { version = "1.0.5", features = [ "full", "visit", "fold", "extra-traits" ] }
+quote = "1.0.2"
+proc-macro2 = "1.0.3"
+Inflector = "0.11.4"
+proc-macro-crate = "0.1.4"
+
+[dev-dependencies]
+runtime-interface = { package = "substrate-runtime-interface", path = ".." }
+codec = { package = "parity-scale-codec", version = "1.0.6", features = [ "derive" ] }
+externalities = { package = "substrate-externalities", path = "../../externalities" }
+rustversion = "1.0.0"
+trybuild = "1.0.17"
+
+# We actually don't need the `std` feature in this crate, but the tests require it.
+[features]
+default = [ "std" ]
+std = []

substrate/primitives/runtime-interface/proc-macro/src/lib.rs

@@ -0,0 +1,265 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! This crate provides procedural macros for usage within the context of the Substrate runtime
+//! interface.
+//!
+//! The following macros are provided:
+//!
+//! 1. The [`#[runtime_interface]`](attr.runtime_interface.html) attribute macro for generating the
+//!    runtime interfaces.
+//! 2. The [`PassByCodec`](derive.PassByCodec.html) derive macro for implementing `PassBy` with `Codec`.
+//! 3. The [`PassByEnum`](derive.PassByInner.html) derive macro for implementing `PassBy` with `Enum`.
+//! 4. The [`PassByInner`](derive.PassByInner.html) derive macro for implementing `PassBy` with `Inner`.
+
+extern crate proc_macro;
+
+use syn::{parse_macro_input, ItemTrait, DeriveInput};
+
+mod pass_by;
+mod runtime_interface;
+mod utils;
+
+/// Attribute macro for transforming a trait declaration into a runtime interface.
+///
+/// A runtime interface is a fixed interface between a Substrate compatible runtime and the native
+/// node. This interface is callable from a native and a wasm runtime. The macro will generate the
+/// corresponding code for the native implementation and the code for calling from the wasm
+/// side to the native implementation.
+///
+/// The macro expects the runtime interface declaration as trait declaration:
+///
+/// ```
+/// # use runtime_interface::runtime_interface;
+///
+/// #[runtime_interface]
+/// trait Interface {
+///     /// A function that can be called from native/wasm.
+///     ///
+///     /// The implementation given to this function is only compiled on native.
+///     fn call_some_complex_code(data: &[u8]) -> Vec<u8> {
+///         // Here you could call some rather complex code that only compiles on native or
+///         // is way faster in native than executing it in wasm.
+///         Vec::new()
+///     }
+///
+///     /// A function can take a `&self` or `&mut self` argument to get access to the
+///     /// `Externalities`. (The generated method does not require
+///     /// this argument, so the function can be called just with the `optional` argument)
+///     fn set_or_clear(&mut self, optional: Option<Vec<u8>>) {
+///         match optional {
+///             Some(value) => self.set_storage([1, 2, 3, 4].to_vec(), value),
+///             None => self.clear_storage(&[1, 2, 3, 4]),
+///         }
+///     }
+/// }
+/// ```
+///
+///
+/// The given example will generate roughly the following code for native:
+///
+/// ```
+/// // The name of the trait is converted to snake case and used as mod name.
+/// //
+/// // Be aware that this module is not `public`, the visibility of the module is determined based
+/// // on the visibility of the trait declaration.
+/// mod interface {
+///     trait Interface {
+///         fn call_some_complex_code(data: &[u8]) -> Vec<u8>;
+///         fn set_or_clear(&mut self, optional: Option<Vec<u8>>);
+///     }
+///
+///     impl Interface for &mut dyn externalities::Externalities {
+///         fn call_some_complex_code(data: &[u8]) -> Vec<u8> { Vec::new() }
+///         fn set_or_clear(&mut self, optional: Option<Vec<u8>>) {
+///             match optional {
+///                 Some(value) => self.set_storage([1, 2, 3, 4].to_vec(), value),
+///                 None => self.clear_storage(&[1, 2, 3, 4]),
+///             }
+///         }
+///     }
+///
+///     pub fn call_some_complex_code(data: &[u8]) -> Vec<u8> {
+///         <&mut dyn externalities::Externalities as Interface>::call_some_complex_code(data)
+///     }
+///
+///     pub fn set_or_clear(optional: Option<Vec<u8>>) {
+///         externalities::with_externalities(|mut ext| Interface::set_or_clear(&mut ext, optional))
+///             .expect("`set_or_clear` called outside of an Externalities-provided environment.")
+///     }
+///
+///     /// This type implements the `HostFunctions` trait (from `substrate-wasm-interface`) and
+///     /// provides the host implementation for the wasm side. The host implementation converts the
+///     /// arguments from wasm to native and calls the corresponding native function.
+///     ///
+///     /// This type needs to be passed to the wasm executor, so that the host functions will be
+///     /// registered in the executor.
+///     pub struct HostFunctions;
+/// }
+/// ```
+///
+///
+/// The given example will generate roughly the following code for wasm:
+///
+/// ```
+/// mod interface {
+///     mod extern_host_functions_impls {
+///         extern "C" {
+///             /// Every function is exported as `ext_TRAIT_NAME_FUNCTION_NAME_version_VERSION`.
+///             ///
+///             /// `TRAIT_NAME` is converted into snake case.
+///             ///
+///             /// The type for each argument of the exported function depends on
+///             /// `<ARGUMENT_TYPE as RIType>::FFIType`.
+///             ///
+///             /// `data` holds the pointer and the length to the `[u8]` slice.
+///             pub fn ext_Interface_call_some_complex_code_version_1(data: u64) -> u64;
+///             /// `optional` holds the pointer and the length of the encoded value.
+///             pub fn ext_Interface_set_or_clear_version_1(optional: u64);
+///         }
+///     }
+///
+///     /// The type is actually `ExchangeableFunction` (from `substrate-runtime-interface`).
+///     ///
+///     /// This can be used to replace the implementation of the `call_some_complex_code` function.
+///     /// Instead of calling into the host, the callee will automatically call the other
+///     /// implementation.
+///     ///
+///     /// To replace the implementation:
+///     ///
+///     /// `host_call_some_complex_code.replace_implementation(some_other_impl)`
+///     pub static host_call_some_complex_code: () = ();
+///     pub static host_set_or_clear: () = ();
+///
+///     pub fn call_some_complex_code(data: &[u8]) -> Vec<u8> {
+///         // This is the actual call: `host_call_some_complex_code.get()(data)`
+///         //
+///         // But that does not work for several reasons in this example, so we just return an
+///         // empty vector.
+///         Vec::new()
+///     }
+///
+///     pub fn set_or_clear(optional: Option<Vec<u8>>) {
+///         // Same as above
+///     }
+/// }
+/// ```
+///
+/// # Argument types
+///
+/// The macro supports any kind of argument type, as long as it implements `RIType` and the required
+/// `FromFFIValue`/`IntoFFIValue` from `substrate-runtime-interface`. The macro will convert each
+/// argument to the corresponding FFI representation and will call into the host using this FFI
+/// representation. On the host each argument is converted back to the native representation and
+/// the native implementation is called. Any return value is handled in the same way.
+///
+/// # Wasm only interfaces
+///
+/// Some interfaces are only required from within the wasm runtime e.g. the allocator interface.
+/// To support this, the macro can be called like `#[runtime_interface(wasm_only)]`. This instructs
+/// the macro to make two significant changes to the generated code:
+///
+/// 1. The generated functions are not callable from the native side.
+/// 2. The trait as shown above is not implemented for `Externalities` and is instead implemented
+///    for `FunctionExecutor` (from `substrate-wasm-interface`).
+#[proc_macro_attribute]
+pub fn runtime_interface(
+	attrs: proc_macro::TokenStream,
+	input: proc_macro::TokenStream,
+) -> proc_macro::TokenStream {
+	let trait_def = parse_macro_input!(input as ItemTrait);
+	let wasm_only = parse_macro_input!(attrs as Option<runtime_interface::keywords::wasm_only>);
+
+	runtime_interface::runtime_interface_impl(trait_def, wasm_only.is_some())
+		.unwrap_or_else(|e| e.to_compile_error())
+		.into()
+}
+
+/// Derive macro for implementing `PassBy` with the `Codec` strategy.
+///
+/// This requires that the type implements `Encode` and `Decode` from `parity-scale-codec`.
+///
+/// # Example
+///
+/// ```
+/// # use runtime_interface::pass_by::PassByCodec;
+/// # use codec::{Encode, Decode};
+/// #[derive(PassByCodec, Encode, Decode)]
+/// struct EncodableType {
+///     name: Vec<u8>,
+///     param: u32,
+/// }
+/// ```
+#[proc_macro_derive(PassByCodec)]
+pub fn pass_by_codec(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+	let input = parse_macro_input!(input as DeriveInput);
+	pass_by::codec_derive_impl(input).unwrap_or_else(|e| e.to_compile_error()).into()
+}
+
+/// Derive macro for implementing `PassBy` with the `Inner` strategy.
+///
+/// Besides implementing `PassBy`, this derive also implements the helper trait `PassByInner`.
+///
+/// The type is required to be a struct with just one field. The field type needs to implement
+/// the required traits to pass it between the wasm and the native side. (See the runtime interface
+/// crate for more information about these traits.)
+///
+/// # Example
+///
+/// ```
+/// # use runtime_interface::pass_by::PassByInner;
+/// #[derive(PassByInner)]
+/// struct Data([u8; 32]);
+/// ```
+///
+/// ```
+/// # use runtime_interface::pass_by::PassByInner;
+/// #[derive(PassByInner)]
+/// struct Data {
+///     data: [u8; 32],
+/// }
+/// ```
+#[proc_macro_derive(PassByInner)]
+pub fn pass_by_inner(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+	let input = parse_macro_input!(input as DeriveInput);
+	pass_by::inner_derive_impl(input).unwrap_or_else(|e| e.to_compile_error()).into()
+}
+
+/// Derive macro for implementing `PassBy` with the `Enum` strategy.
+///
+/// Besides implementing `PassBy`, this derive also implements `TryFrom<u8>` and `From<Self> for u8`
+/// for the type.
+///
+/// The type is required to be an enum with only unit variants and at maximum `256` variants. Also
+/// it is required that the type implements `Copy`.
+///
+/// # Example
+///
+/// ```
+/// # use runtime_interface::pass_by::PassByEnum;
+/// #[derive(PassByEnum, Copy, Clone)]
+/// enum Data {
+///     Okay,
+///     NotOkay,
+///     // This will not work with the derive.
+///     //Why(u32),
+/// }
+/// ```
+#[proc_macro_derive(PassByEnum)]
+pub fn pass_by_enum(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+	let input = parse_macro_input!(input as DeriveInput);
+	pass_by::enum_derive_impl(input).unwrap_or_else(|e| e.to_compile_error()).into()
+}

substrate/primitives/runtime-interface/proc-macro/src/pass_by/codec.rs

@@ -0,0 +1,58 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Derive macro implementation of `PassBy` with the associated type set to `Codec`.
+//!
+//! It is required that the type implements `Encode` and `Decode` from the `parity-scale-codec`
+//! crate.
+
+use crate::utils::{generate_crate_access, generate_runtime_interface_include};
+
+use syn::{DeriveInput, Result, Generics, parse_quote};
+
+use quote::quote;
+
+use proc_macro2::TokenStream;
+
+/// The derive implementation for `PassBy` with `Codec`.
+pub fn derive_impl(mut input: DeriveInput) -> Result<TokenStream> {
+	add_trait_bounds(&mut input.generics);
+	let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl();
+	let crate_include = generate_runtime_interface_include();
+	let crate_ = generate_crate_access();
+	let ident = input.ident;
+
+	let res = quote! {
+		const _: () = {
+			#crate_include
+
+			impl #impl_generics #crate_::pass_by::PassBy for #ident #ty_generics #where_clause {
+				type PassBy = #crate_::pass_by::Codec<#ident>;
+			}
+		};
+	};
+
+	Ok(res)
+}
+
+/// Add the `codec::Codec` trait bound to every type parameter.
+fn add_trait_bounds(generics: &mut Generics) {
+	let crate_ = generate_crate_access();
+
+	generics.type_params_mut()
+		.for_each(|type_param| type_param.bounds.push(parse_quote!(#crate_::codec::Codec)));
+}
+

substrate/primitives/runtime-interface/proc-macro/src/pass_by/enum_.rs

@@ -0,0 +1,101 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Derive macro implementation of `PassBy` with the associated type set to `Enum`.
+//!
+//! Besides `PassBy`, `TryFrom<u8>` and `From<Self> for u8` are implemented for the type.
+
+use crate::utils::{generate_crate_access, generate_runtime_interface_include};
+
+use syn::{DeriveInput, Result, Data, Fields, Error, Ident};
+
+use quote::quote;
+
+use proc_macro2::{TokenStream, Span};
+
+/// The derive implementation for `PassBy` with `Enum`.
+pub fn derive_impl(input: DeriveInput) -> Result<TokenStream> {
+	let crate_include = generate_runtime_interface_include();
+	let crate_ = generate_crate_access();
+	let ident = input.ident;
+	let enum_fields = get_enum_field_idents(&input.data)?
+		.enumerate()
+		.map(|(i, v)| {
+			let i = i as u8;
+
+			v.map(|v| (quote!(#i => Ok(#ident::#v)), quote!(#ident::#v => #i)))
+		})
+		.collect::<Result<Vec<_>>>()?;
+	let try_from_variants = enum_fields.iter().map(|i| &i.0);
+	let into_variants = enum_fields.iter().map(|i| &i.1);
+
+	let res = quote! {
+		const _: () = {
+			#crate_include
+
+			impl #crate_::pass_by::PassBy for #ident {
+				type PassBy = #crate_::pass_by::Enum<#ident>;
+			}
+
+			impl #crate_::rstd::convert::TryFrom<u8> for #ident {
+				type Error = ();
+
+				fn try_from(inner: u8) -> #crate_::rstd::result::Result<Self, ()> {
+					match inner {
+						#( #try_from_variants, )*
+						_ => Err(()),
+					}
+				}
+			}
+
+			impl From<#ident> for u8 {
+				fn from(var: #ident) -> u8 {
+					match var {
+						#( #into_variants ),*
+					}
+				}
+			}
+		};
+	};
+
+	Ok(res)
+}
+
+/// Get the enum fields idents of the given `data` object as iterator.
+///
+/// Returns an error if the number of variants is greater than `256`, the given `data` is not an
+/// enum or a variant is not an unit.
+fn get_enum_field_idents<'a>(data: &'a Data) -> Result<impl Iterator<Item = Result<&'a Ident>>> {
+	match data {
+		Data::Enum(d) => {
+			if d.variants.len() <= 256 {
+				Ok(
+					d.variants.iter().map(|v| if let Fields::Unit = v.fields {
+						Ok(&v.ident)
+					} else {
+						Err(Error::new(
+							Span::call_site(),
+							"`PassByEnum` only supports unit variants.",
+						))
+					})
+				)
+			} else {
+				Err(Error::new(Span::call_site(), "`PassByEnum` only supports `256` variants."))
+			}
+		},
+		_ => Err(Error::new(Span::call_site(), "`PassByEnum` only supports enums as input type."))
+	}
+}

substrate/primitives/runtime-interface/proc-macro/src/pass_by/inner.rs

@@ -0,0 +1,110 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Derive macro implementation of `PassBy` with the associated type set to `Inner` and of the
+//! helper trait `PassByInner`.
+//!
+//! It is required that the type is a newtype struct, otherwise an error is generated.
+
+use crate::utils::{generate_crate_access, generate_runtime_interface_include};
+
+use syn::{DeriveInput, Result, Generics, parse_quote, Type, Data, Error, Fields, Ident};
+
+use quote::quote;
+
+use proc_macro2::{TokenStream, Span};
+
+/// The derive implementation for `PassBy` with `Inner` and `PassByInner`.
+pub fn derive_impl(mut input: DeriveInput) -> Result<TokenStream> {
+	add_trait_bounds(&mut input.generics);
+	let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl();
+	let crate_include = generate_runtime_interface_include();
+	let crate_ = generate_crate_access();
+	let ident = input.ident;
+	let (inner_ty, inner_name) = extract_inner_ty_and_name(&input.data)?;
+
+	let access_inner = match inner_name {
+		Some(ref name) => quote!(self.#name),
+		None => quote!(self.0),
+	};
+
+	let from_inner = match inner_name {
+		Some(name) => quote!(Self { #name: inner }),
+		None => quote!(Self(inner)),
+	};
+
+	let res = quote! {
+		const _: () = {
+			#crate_include
+
+			impl #impl_generics #crate_::pass_by::PassBy for #ident #ty_generics #where_clause {
+				type PassBy = #crate_::pass_by::Inner<#ident, #inner_ty>;
+			}
+
+			impl #impl_generics #crate_::pass_by::PassByInner for #ident #ty_generics #where_clause {
+				type Inner = #inner_ty;
+
+				fn into_inner(self) -> Self::Inner {
+					#access_inner
+				}
+
+				fn inner(&self) -> &Self::Inner {
+					&#access_inner
+				}
+
+				fn from_inner(inner: Self::Inner) -> Self {
+					#from_inner
+				}
+			}
+		};
+	};
+
+	Ok(res)
+}
+
+/// Add the `RIType` trait bound to every type parameter.
+fn add_trait_bounds(generics: &mut Generics) {
+	let crate_ = generate_crate_access();
+
+	generics.type_params_mut()
+		.for_each(|type_param| type_param.bounds.push(parse_quote!(#crate_::RIType)));
+}
+
+/// Extract the inner type and optional name from given input data.
+///
+/// It also checks that the input data is a newtype struct.
+fn extract_inner_ty_and_name(data: &Data) -> Result<(Type, Option<Ident>)> {
+	if let Data::Struct(ref struct_data) = data {
+		match struct_data.fields {
+			Fields::Named(ref named) if named.named.len() == 1 => {
+				let field = &named.named[0];
+				return Ok((field.ty.clone(), field.ident.clone()))
+			},
+			Fields::Unnamed(ref unnamed) if unnamed.unnamed.len() == 1 => {
+				let field = &unnamed.unnamed[0];
+				return Ok((field.ty.clone(), field.ident.clone()))
+			}
+			_ => {},
+		}
+	}
+
+	Err(
+		Error::new(
+			Span::call_site(),
+			"Only newtype/one field structs are supported by `PassByInner`!",
+		)
+	)
+}

substrate/primitives/runtime-interface/proc-macro/src/pass_by/mod.rs

@@ -0,0 +1,25 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! All the `PassBy*` derive implementations.
+
+mod codec;
+mod enum_;
+mod inner;
+
+pub use self::codec::derive_impl as codec_derive_impl;
+pub use enum_::derive_impl as enum_derive_impl;
+pub use inner::derive_impl as inner_derive_impl;

substrate/primitives/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs

@@ -0,0 +1,186 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Generates the bare function interface for a given trait definition.
+//!
+//! The bare functions are the ones that will be called by the user. On the native/host side, these
+//! functions directly execute the provided implementation. On the wasm side, these
+//! functions will prepare the parameters for the FFI boundary, call the external host function
+//! exported into wasm and convert back the result.
+//!
+//! [`generate`](bare_function_interface::generate) is the entry point for generating for each
+//! trait method one bare function.
+//!
+//! [`function_for_method`](bare_function_interface::function_for_method) generates the bare
+//! function per trait method. Each bare function contains both implementations. The implementations
+//! are feature-gated, so that one is compiled for the native and the other for the wasm side.
+
+use crate::utils::{
+	generate_crate_access, create_exchangeable_host_function_ident, get_function_arguments,
+	get_function_argument_names, get_trait_methods,
+};
+
+use syn::{
+	Ident, ItemTrait, TraitItemMethod, FnArg, Signature, Result, spanned::Spanned, parse_quote,
+};
+
+use proc_macro2::{TokenStream, Span};
+
+use quote::{quote, quote_spanned};
+
+use std::iter;
+
+/// Generate one bare function per trait method. The name of the bare function is equal to the name
+/// of the trait method.
+pub fn generate(trait_def: &ItemTrait, is_wasm_only: bool) -> Result<TokenStream> {
+	let trait_name = &trait_def.ident;
+	get_trait_methods(trait_def).try_fold(TokenStream::new(), |mut t, m| {
+		t.extend(function_for_method(trait_name, m, is_wasm_only)?);
+		Ok(t)
+	})
+}
+
+/// Generates the bare function implementation for the given method for the host and wasm side.
+fn function_for_method(
+	trait_name: &Ident,
+	method: &TraitItemMethod,
+	is_wasm_only: bool,
+) -> Result<TokenStream> {
+	let std_impl = function_std_impl(trait_name, method, is_wasm_only)?;
+	let no_std_impl = function_no_std_impl(method)?;
+
+	Ok(
+		quote! {
+			#std_impl
+
+			#no_std_impl
+		}
+	)
+}
+
+/// Generates the bare function implementation for `cfg(not(feature = "std"))`.
+fn function_no_std_impl(method: &TraitItemMethod) -> Result<TokenStream> {
+	let function_name = &method.sig.ident;
+	let host_function_name = create_exchangeable_host_function_ident(&method.sig.ident);
+	let args = get_function_arguments(&method.sig);
+	let arg_names = get_function_argument_names(&method.sig);
+	let return_value = &method.sig.output;
+	let attrs = &method.attrs;
+
+	Ok(
+		quote! {
+			#[cfg(not(feature = "std"))]
+			#( #attrs )*
+			pub fn #function_name( #( #args, )* ) #return_value {
+				// Call the host function
+				#host_function_name.get()( #( #arg_names, )* )
+			}
+		}
+	)
+}
+
+/// Generates the bare function implementation for `cfg(feature = "std")`.
+fn function_std_impl(
+	trait_name: &Ident,
+	method: &TraitItemMethod,
+	is_wasm_only: bool,
+) -> Result<TokenStream> {
+	let function_name = &method.sig.ident;
+	let crate_ = generate_crate_access();
+	let args = get_function_arguments(&method.sig).map(FnArg::Typed).chain(
+		// Add the function context as last parameter when this is a wasm only interface.
+		iter::from_fn(||
+			if is_wasm_only {
+				Some(
+					parse_quote!(
+						mut __function_context__: &mut dyn #crate_::wasm_interface::FunctionContext
+					)
+				)
+			} else {
+				None
+			}
+		).take(1),
+	);
+	let return_value = &method.sig.output;
+	let attrs = &method.attrs;
+	// Don't make the function public accessible when this is a wasm only interface.
+	let vis = if is_wasm_only { quote!() } else { quote!(pub) };
+	let call_to_trait = generate_call_to_trait(trait_name, method, is_wasm_only);
+
+	Ok(
+		quote_spanned! { method.span() =>
+			#[cfg(feature = "std")]
+			#( #attrs )*
+			#vis fn #function_name( #( #args, )* ) #return_value {
+				#call_to_trait
+			}
+		}
+	)
+}
+
+/// Generate the call to the interface trait.
+fn generate_call_to_trait(
+	trait_name: &Ident,
+	method: &TraitItemMethod,
+	is_wasm_only: bool,
+) -> TokenStream {
+	let crate_ = generate_crate_access();
+	let method_name = &method.sig.ident;
+	let expect_msg = format!(
+		"`{}` called outside of an Externalities-provided environment.",
+		method_name,
+	);
+	let arg_names = get_function_argument_names(&method.sig);
+
+	if takes_self_argument(&method.sig) {
+		let instance = if is_wasm_only {
+			Ident::new("__function_context__", Span::call_site())
+		} else {
+			Ident::new("__externalities__", Span::call_site())
+		};
+
+		let impl_ = quote!( #trait_name::#method_name(&mut #instance, #( #arg_names, )*) );
+
+		if is_wasm_only {
+			quote_spanned! { method.span() => #impl_ }
+		} else {
+			quote_spanned! { method.span() =>
+				#crate_::with_externalities(|mut #instance| #impl_).expect(#expect_msg)
+			}
+		}
+	} else {
+		// The name of the trait the interface trait is implemented for
+		let impl_trait_name = if is_wasm_only {
+			quote!( #crate_::wasm_interface::FunctionContext )
+		} else {
+			quote!( #crate_::Externalities )
+		};
+
+		quote_spanned! { method.span() =>
+			<&mut dyn #impl_trait_name as #trait_name>::#method_name(
+				#( #arg_names, )*
+			)
+		}
+	}
+}
+
+/// Returns if the given `Signature` takes a `self` argument.
+fn takes_self_argument(sig: &Signature) -> bool {
+	match sig.inputs.first() {
+		Some(FnArg::Receiver(_)) => true,
+		_ => false,
+	}
+}

substrate/primitives/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs

@@ -0,0 +1,415 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Generates the extern host functions and the implementation for these host functions.
+//!
+//! The extern host functions will be called by the bare function interface from the Wasm side.
+//! The implementation of these host functions will be called on the host side from the Wasm
+//! executor. These implementations call the bare function interface.
+
+use crate::utils::{
+	generate_crate_access, create_host_function_ident, get_function_argument_names,
+	get_function_argument_types_without_ref, get_function_argument_types_ref_and_mut,
+	get_function_argument_names_and_types_without_ref, get_trait_methods, get_function_arguments,
+	get_function_argument_types, create_exchangeable_host_function_ident,
+};
+
+use syn::{
+	ItemTrait, TraitItemMethod, Result, ReturnType, Ident, TraitItem, Pat, Error, Signature,
+	spanned::Spanned,
+};
+
+use proc_macro2::{TokenStream, Span};
+
+use quote::{quote, ToTokens};
+
+use inflector::Inflector;
+
+use std::iter::{Iterator, self};
+
+/// Generate the extern host functions for wasm and the `HostFunctions` struct that provides the
+/// implementations for the host functions on the host.
+pub fn generate(trait_def: &ItemTrait, is_wasm_only: bool) -> Result<TokenStream> {
+	let trait_name = &trait_def.ident;
+	let extern_host_function_impls = get_trait_methods(trait_def)
+		.try_fold(TokenStream::new(), |mut t, m| {
+			t.extend(generate_extern_host_function(m, trait_name)?);
+			Ok::<_, Error>(t)
+		})?;
+	let exchangeable_host_functions = get_trait_methods(trait_def)
+		.try_fold(TokenStream::new(), |mut t, m| {
+			t.extend(generate_exchangeable_host_function(m)?);
+			Ok::<_, Error>(t)
+		})?;
+	let host_functions_struct = generate_host_functions_struct(trait_def, is_wasm_only)?;
+
+	Ok(
+		quote! {
+			/// The implementations of the extern host functions. This special implementation module
+			/// is required to change the extern host functions signature to
+			/// `unsafe fn name(args) -> ret` to make the function implementations exchangeable.
+			#[cfg(not(feature = "std"))]
+			mod extern_host_function_impls {
+				use super::*;
+
+				#extern_host_function_impls
+			}
+
+			#exchangeable_host_functions
+
+			#host_functions_struct
+		}
+	)
+}
+
+/// Generate the extern host function for the given method.
+fn generate_extern_host_function(method: &TraitItemMethod, trait_name: &Ident) -> Result<TokenStream> {
+	let crate_ = generate_crate_access();
+	let args = get_function_arguments(&method.sig);
+	let arg_types = get_function_argument_types_without_ref(&method.sig);
+	let arg_types2 = get_function_argument_types_without_ref(&method.sig);
+	let arg_names = get_function_argument_names(&method.sig);
+	let arg_names2 = get_function_argument_names(&method.sig);
+	let arg_names3 = get_function_argument_names(&method.sig);
+	let function = &method.sig.ident;
+	let ext_function = create_host_function_ident(&method.sig.ident, trait_name);
+	let doc_string = format!(
+		" Default extern host function implementation for [`super::{}`].",
+		method.sig.ident,
+	);
+	let return_value = &method.sig.output;
+
+	let ffi_return_value = match method.sig.output {
+		ReturnType::Default => quote!(),
+		ReturnType::Type(_, ref ty) => quote! {
+			-> <#ty as #crate_::RIType>::FFIType
+		},
+	};
+
+	let convert_return_value = match return_value {
+		ReturnType::Default => quote!(),
+		ReturnType::Type(_, ref ty) => quote! {
+			<#ty as #crate_::wasm::FromFFIValue>::from_ffi_value(result)
+		}
+	};
+
+	Ok(
+		quote! {
+			#[doc = #doc_string]
+			pub fn #function ( #( #args ),* ) #return_value {
+				extern "C" {
+					/// The extern function.
+					pub fn #ext_function (
+						#( #arg_names: <#arg_types as #crate_::RIType>::FFIType ),*
+					) #ffi_return_value;
+				}
+
+				// Generate all wrapped ffi values.
+				#(
+					let #arg_names2 = <#arg_types2 as #crate_::wasm::IntoFFIValue>::into_ffi_value(
+						&#arg_names2,
+					);
+				)*
+
+				let result = unsafe { #ext_function( #( #arg_names3.get() ),* ) };
+
+				#convert_return_value
+			}
+		}
+	)
+}
+
+/// Generate the host exchangeable function for the given method.
+fn generate_exchangeable_host_function(method: &TraitItemMethod) -> Result<TokenStream> {
+	let crate_ = generate_crate_access();
+	let arg_types = get_function_argument_types(&method.sig);
+	let function = &method.sig.ident;
+	let exchangeable_function = create_exchangeable_host_function_ident(&method.sig.ident);
+	let doc_string = format!(" Exchangeable host function used by [`{}`].", method.sig.ident);
+	let output = &method.sig.output;
+
+	Ok(
+		quote! {
+			#[cfg(not(feature = "std"))]
+			#[allow(non_upper_case_globals)]
+			#[doc = #doc_string]
+			pub static #exchangeable_function : #crate_::wasm::ExchangeableFunction<
+				fn ( #( #arg_types ),* ) #output
+			> = #crate_::wasm::ExchangeableFunction::new(extern_host_function_impls::#function);
+		}
+	)
+}
+
+/// Generate the `HostFunctions` struct that implements `wasm-interface::HostFunctions` to provide
+/// implementations for the extern host functions.
+fn generate_host_functions_struct(trait_def: &ItemTrait, is_wasm_only: bool) -> Result<TokenStream> {
+	let crate_ = generate_crate_access();
+	let host_functions = trait_def
+		.items
+		.iter()
+		.filter_map(|i| match i {
+			TraitItem::Method(ref method) => Some(method),
+			_ => None,
+		})
+		.map(|m| generate_host_function_implementation(&trait_def.ident, m, is_wasm_only))
+		.collect::<Result<Vec<_>>>()?;
+
+	Ok(
+		quote! {
+			/// Provides implementations for the extern host functions.
+			#[cfg(feature = "std")]
+			pub struct HostFunctions;
+
+			#[cfg(feature = "std")]
+			impl #crate_::wasm_interface::HostFunctions for HostFunctions {
+				fn host_functions() -> Vec<&'static dyn #crate_::wasm_interface::Function> {
+					vec![ #( #host_functions ),* ]
+				}
+			}
+		}
+	)
+}
+
+/// Generates the host function struct that implements `wasm_interface::Function` and returns a static
+/// reference to this struct.
+///
+/// When calling from wasm into the host, we will call the `execute` function that calls the native
+/// implementation of the function.
+fn generate_host_function_implementation(
+	trait_name: &Ident,
+	method: &TraitItemMethod,
+	is_wasm_only: bool,
+) -> Result<TokenStream> {
+	let name = create_host_function_ident(&method.sig.ident, trait_name).to_string();
+	let struct_name = Ident::new(&name.to_pascal_case(), Span::call_site());
+	let crate_ = generate_crate_access();
+	let signature = generate_wasm_interface_signature_for_host_function(&method.sig)?;
+	let wasm_to_ffi_values = generate_wasm_to_ffi_values(
+		&method.sig,
+		trait_name,
+	).collect::<Result<Vec<_>>>()?;
+	let ffi_to_host_values = generate_ffi_to_host_value(&method.sig).collect::<Result<Vec<_>>>()?;
+	let host_function_call = generate_host_function_call(&method.sig, is_wasm_only);
+	let into_preallocated_ffi_value = generate_into_preallocated_ffi_value(&method.sig)?;
+	let convert_return_value = generate_return_value_into_wasm_value(&method.sig);
+
+	Ok(
+		quote! {
+			{
+				struct #struct_name;
+
+				#[allow(unused)]
+				impl #crate_::wasm_interface::Function for #struct_name {
+					fn name(&self) -> &str {
+						#name
+					}
+
+					fn signature(&self) -> #crate_::wasm_interface::Signature {
+						#signature
+					}
+
+					fn execute(
+						&self,
+						__function_context__: &mut dyn #crate_::wasm_interface::FunctionContext,
+						args: &mut dyn Iterator<Item = #crate_::wasm_interface::Value>,
+					) -> std::result::Result<Option<#crate_::wasm_interface::Value>, String> {
+						#( #wasm_to_ffi_values )*
+						#( #ffi_to_host_values )*
+						#host_function_call
+						#into_preallocated_ffi_value
+						#convert_return_value
+					}
+				}
+
+				&#struct_name as &dyn #crate_::wasm_interface::Function
+			}
+		}
+	)
+}
+
+/// Generate the `wasm_interface::Signature` for the given host function `sig`.
+fn generate_wasm_interface_signature_for_host_function(sig: &Signature) -> Result<TokenStream> {
+	let crate_ = generate_crate_access();
+	let return_value = match &sig.output {
+		ReturnType::Type(_, ty) =>
+			quote! {
+				Some( <<#ty as #crate_::RIType>::FFIType as #crate_::wasm_interface::IntoValue>::VALUE_TYPE )
+			},
+		ReturnType::Default => quote!( None ),
+	};
+	let arg_types = get_function_argument_types_without_ref(sig)
+		.map(|ty| quote! {
+			<<#ty as #crate_::RIType>::FFIType as #crate_::wasm_interface::IntoValue>::VALUE_TYPE
+		});
+
+	Ok(
+		quote! {
+			#crate_::wasm_interface::Signature {
+				args: std::borrow::Cow::Borrowed(&[ #( #arg_types ),* ][..]),
+				return_value: #return_value,
+			}
+		}
+	)
+}
+
+/// Generate the code that converts the wasm values given to `HostFunctions::execute` into the FFI
+/// values.
+fn generate_wasm_to_ffi_values<'a>(
+	sig: &'a Signature,
+	trait_name: &'a Ident,
+) -> impl Iterator<Item = Result<TokenStream>> + 'a {
+	let crate_ = generate_crate_access();
+	let function_name = &sig.ident;
+	let error_message = format!(
+		"Number of arguments given to `{}` does not match the expected number of arguments!",
+		function_name,
+	);
+
+	get_function_argument_names_and_types_without_ref(sig)
+		.map(move |(name, ty)| {
+			let try_from_error = format!(
+				"Could not instantiate `{}` from wasm value while executing `{}` from interface `{}`!",
+				name.to_token_stream(),
+				function_name,
+				trait_name,
+			);
+
+			let var_name = generate_ffi_value_var_name(&name)?;
+
+			Ok(quote! {
+				let val = args.next().ok_or_else(|| #error_message)?;
+				let #var_name = <
+					<#ty as #crate_::RIType>::FFIType as #crate_::wasm_interface::TryFromValue
+				>::try_from_value(val).ok_or_else(|| #try_from_error)?;
+			})
+		})
+}
+
+/// Generate the code to convert the ffi values on the host to the host values using `FromFFIValue`.
+fn generate_ffi_to_host_value<'a>(
+	sig: &'a Signature,
+) -> impl Iterator<Item = Result<TokenStream>> + 'a {
+	let mut_access = get_function_argument_types_ref_and_mut(sig);
+	let crate_ = generate_crate_access();
+
+	get_function_argument_names_and_types_without_ref(sig)
+		.zip(mut_access.map(|v| v.and_then(|m| m.1)))
+		.map(move |((name, ty), mut_access)| {
+			let ffi_value_var_name = generate_ffi_value_var_name(&name)?;
+
+			Ok(
+				quote! {
+					let #mut_access #name = <#ty as #crate_::host::FromFFIValue>::from_ffi_value(
+						__function_context__,
+						#ffi_value_var_name,
+					)?;
+				}
+			)
+		})
+}
+
+/// Generate the code to call the host function and the ident that stores the result.
+fn generate_host_function_call(sig: &Signature, is_wasm_only: bool) -> TokenStream {
+	let host_function_name = &sig.ident;
+	let result_var_name = generate_host_function_result_var_name(&sig.ident);
+	let ref_and_mut = get_function_argument_types_ref_and_mut(sig).map(|ram|
+		ram.map(|(vr, vm)| quote!(#vr #vm))
+	);
+	let names = get_function_argument_names(sig);
+
+	let var_access = names.zip(ref_and_mut)
+		.map(|(n, ref_and_mut)| {
+			quote!( #ref_and_mut #n )
+		})
+		// If this is a wasm only interface, we add the function context as last parameter.
+		.chain(
+			iter::from_fn(|| if is_wasm_only { Some(quote!(__function_context__)) } else { None })
+				.take(1)
+		);
+
+	quote! {
+		let #result_var_name = #host_function_name ( #( #var_access ),* );
+	}
+}
+
+/// Generate the variable name that stores the result of the host function.
+fn generate_host_function_result_var_name(name: &Ident) -> Ident {
+	Ident::new(&format!("{}_result", name), Span::call_site())
+}
+
+/// Generate the variable name that stores the FFI value.
+fn generate_ffi_value_var_name(pat: &Pat) -> Result<Ident> {
+	match pat {
+		Pat::Ident(pat_ident) => {
+			if let Some(by_ref) = pat_ident.by_ref {
+				Err(Error::new(by_ref.span(), "`ref` not supported!"))
+			} else if let Some(sub_pattern) = &pat_ident.subpat {
+				Err(Error::new(sub_pattern.0.span(), "Not supported!"))
+			} else {
+				Ok(Ident::new(&format!("{}_ffi_value", pat_ident.ident), Span::call_site()))
+			}
+		}
+		_ => Err(Error::new(pat.span(), "Not supported as variable name!"))
+	}
+}
+
+/// Generate code that copies data from the host back to preallocated wasm memory.
+///
+/// Any argument that is given as `&mut` is interpreted as preallocated memory and it is expected
+/// that the type implements `IntoPreAllocatedFFIValue`.
+fn generate_into_preallocated_ffi_value(sig: &Signature) -> Result<TokenStream> {
+	let crate_ = generate_crate_access();
+	let ref_and_mut = get_function_argument_types_ref_and_mut(sig).map(|ram|
+		ram.and_then(|(vr, vm)| vm.map(|v| (vr, v)))
+	);
+	let names_and_types = get_function_argument_names_and_types_without_ref(sig);
+
+	ref_and_mut.zip(names_and_types)
+		.filter_map(|(ram, (name, ty))| ram.map(|_| (name, ty)))
+		.map(|(name, ty)| {
+			let ffi_var_name = generate_ffi_value_var_name(&name)?;
+
+			Ok(
+				quote! {
+					<#ty as #crate_::host::IntoPreallocatedFFIValue>::into_preallocated_ffi_value(
+						#name,
+						__function_context__,
+						#ffi_var_name,
+					)?;
+				}
+			)
+		})
+		.collect()
+}
+
+/// Generate the code that converts the return value into the appropriate wasm value.
+fn generate_return_value_into_wasm_value(sig: &Signature) -> TokenStream {
+	let crate_ = generate_crate_access();
+
+	match &sig.output {
+		ReturnType::Default => quote!( Ok(None) ),
+		ReturnType::Type(_, ty) => {
+			let result_var_name = generate_host_function_result_var_name(&sig.ident);
+
+			quote! {
+				<#ty as #crate_::host::IntoFFIValue>::into_ffi_value(
+					#result_var_name,
+					__function_context__,
+				).map(#crate_::wasm_interface::IntoValue::into_value).map(Some)
+			}
+		}
+	}
+}

substrate/primitives/runtime-interface/proc-macro/src/runtime_interface/mod.rs

@@ -0,0 +1,65 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::utils::generate_runtime_interface_include;
+
+use proc_macro2::{Span, TokenStream};
+
+use syn::{Ident, ItemTrait, Result};
+
+use inflector::Inflector;
+
+use quote::quote;
+
+mod bare_function_interface;
+mod host_function_interface;
+mod trait_decl_impl;
+
+/// Custom keywords supported by the `runtime_interface` attribute.
+pub mod keywords {
+	// Custom keyword `wasm_only` that can be given as attribute to [`runtime_interface`].
+	syn::custom_keyword!(wasm_only);
+}
+
+/// Implementation of the `runtime_interface` attribute.
+///
+/// It expects the trait definition the attribute was put above and if this should be an wasm only
+/// interface.
+pub fn runtime_interface_impl(trait_def: ItemTrait, is_wasm_only: bool) -> Result<TokenStream> {
+	let bare_functions = bare_function_interface::generate(&trait_def, is_wasm_only)?;
+	let crate_include = generate_runtime_interface_include();
+	let mod_name = Ident::new(&trait_def.ident.to_string().to_snake_case(), Span::call_site());
+	let trait_decl_impl = trait_decl_impl::process(&trait_def, is_wasm_only)?;
+	let host_functions = host_function_interface::generate(&trait_def, is_wasm_only)?;
+	let vis = trait_def.vis;
+	let attrs = &trait_def.attrs;
+
+	let res = quote! {
+		#( #attrs )*
+		#vis mod #mod_name {
+			use super::*;
+			#crate_include
+
+			#bare_functions
+
+			#trait_decl_impl
+
+			#host_functions
+		}
+	};
+
+	Ok(res)
+}

substrate/primitives/runtime-interface/proc-macro/src/runtime_interface/trait_decl_impl.rs

@@ -0,0 +1,146 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Checks the trait declaration, makes the trait declaration module local, removes all method
+//! default implementations and implements the trait for `&mut dyn Externalities`.
+
+use crate::utils::{generate_crate_access, get_function_argument_types_without_ref};
+
+use syn::{
+	ItemTrait, TraitItemMethod, Result, TraitItem, Error, fold::{self, Fold}, spanned::Spanned,
+	Visibility, Receiver, Type, Generics,
+};
+
+use proc_macro2::TokenStream;
+
+use quote::quote;
+
+/// Process the given trait definition, by checking that the definition is valid, fold it to the
+/// essential definition and implement this essential definition for `dyn Externalities`.
+pub fn process(trait_def: &ItemTrait, is_wasm_only: bool) -> Result<TokenStream> {
+	let impl_trait = impl_trait_for_externalities(trait_def, is_wasm_only)?;
+	let essential_trait_def = ToEssentialTraitDef::convert(trait_def.clone())?;
+
+	Ok(
+		quote! {
+			#impl_trait
+
+			#essential_trait_def
+		}
+	)
+}
+
+/// Converts the given trait definition into the essential trait definition without method
+/// default implementations and visibility set to inherited.
+struct ToEssentialTraitDef {
+	/// All errors found while doing the conversion.
+	errors: Vec<Error>,
+}
+
+impl ToEssentialTraitDef {
+	/// Convert the given trait definition to the essential trait definition.
+	fn convert(trait_def: ItemTrait) -> Result<ItemTrait> {
+		let mut folder = ToEssentialTraitDef {
+			errors: Vec::new(),
+		};
+
+		let res = folder.fold_item_trait(trait_def);
+
+		if let Some(first_error) = folder.errors.pop() {
+			Err(
+				folder.errors.into_iter().fold(first_error, |mut o, n| {
+					o.combine(n);
+					o
+				})
+			)
+		} else {
+			Ok(res)
+		}
+	}
+
+	fn push_error<S: Spanned>(&mut self, span: &S, msg: &str) {
+		self.errors.push(Error::new(span.span(), msg));
+	}
+
+	fn error_on_generic_parameters(&mut self, generics: &Generics) {
+		if let Some(param) = generics.params.first() {
+			self.push_error(param, "Generic parameters not supported.");
+		}
+	}
+}
+
+impl Fold for ToEssentialTraitDef {
+	fn fold_trait_item_method(&mut self, mut method: TraitItemMethod) -> TraitItemMethod {
+		if method.default.take().is_none() {
+			self.push_error(&method, "Methods need to have an implementation.");
+		}
+
+		let arg_types = get_function_argument_types_without_ref(&method.sig);
+		arg_types.filter_map(|ty|
+			match *ty {
+				Type::ImplTrait(impl_trait) => Some(impl_trait),
+				_ => None
+			}
+		).for_each(|invalid| self.push_error(&invalid, "`impl Trait` syntax not supported."));
+
+		self.error_on_generic_parameters(&method.sig.generics);
+
+		fold::fold_trait_item_method(self, method)
+	}
+
+	fn fold_item_trait(&mut self, mut trait_def: ItemTrait) -> ItemTrait {
+		self.error_on_generic_parameters(&trait_def.generics);
+
+		trait_def.vis = Visibility::Inherited;
+		fold::fold_item_trait(self, trait_def)
+	}
+
+	fn fold_receiver(&mut self, receiver: Receiver) -> Receiver {
+		if receiver.reference.is_none() {
+			self.push_error(&receiver, "Taking `Self` by value is not allowed.");
+		}
+
+		fold::fold_receiver(self, receiver)
+	}
+}
+
+/// Implements the given trait definition for `dyn Externalities`.
+fn impl_trait_for_externalities(trait_def: &ItemTrait, is_wasm_only: bool) -> Result<TokenStream> {
+	let trait_ = &trait_def.ident;
+	let crate_ = generate_crate_access();
+	let methods = trait_def
+		.items
+		.iter()
+		.filter_map(|i| match i {
+			TraitItem::Method(ref method) => Some(method),
+			_ => None,
+		});
+
+	let impl_type = if is_wasm_only {
+		quote!( &mut dyn #crate_::wasm_interface::FunctionContext )
+	} else {
+		quote!( &mut dyn #crate_::Externalities )
+	};
+
+	Ok(
+		quote! {
+			#[cfg(feature = "std")]
+			impl #trait_ for #impl_type {
+				#( #methods )*
+			}
+		}
+	)
+}

substrate/primitives/runtime-interface/proc-macro/src/utils.rs

@@ -0,0 +1,162 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>
+
+//! Util function used by this crate.
+
+use proc_macro2::{TokenStream, Span};
+
+use syn::{
+	Ident, Error, Signature, Pat, PatType, FnArg, Type, token, TraitItemMethod, ItemTrait,
+	TraitItem, parse_quote, spanned::Spanned,
+};
+
+use proc_macro_crate::crate_name;
+
+use std::env;
+
+use quote::quote;
+
+use inflector::Inflector;
+
+/// Generates the include for the runtime-interface crate.
+pub fn generate_runtime_interface_include() -> TokenStream {
+	if env::var("CARGO_PKG_NAME").unwrap() == "substrate-runtime-interface" {
+		TokenStream::new()
+	} else {
+		match crate_name("substrate-runtime-interface") {
+			Ok(crate_name) => {
+				let crate_name = Ident::new(&crate_name, Span::call_site());
+				quote!(
+					#[doc(hidden)]
+					extern crate #crate_name as proc_macro_runtime_interface;
+				)
+			},
+			Err(e) => {
+				let err = Error::new(Span::call_site(), &e).to_compile_error();
+				quote!( #err )
+			}
+		}
+	}
+}
+
+/// Generates the access to the `substrate-runtime-interface` crate.
+pub fn generate_crate_access() -> TokenStream {
+	if env::var("CARGO_PKG_NAME").unwrap() == "substrate-runtime-interface" {
+		quote!( substrate_runtime_interface )
+	} else {
+		quote!( proc_macro_runtime_interface )
+	}
+}
+
+/// Create the exchangeable host function identifier for the given function name.
+pub fn create_exchangeable_host_function_ident(name: &Ident) -> Ident {
+	Ident::new(&format!("host_{}", name), Span::call_site())
+}
+
+/// Create the host function identifier for the given function name.
+pub fn create_host_function_ident(name: &Ident, trait_name: &Ident) -> Ident {
+	Ident::new(
+		&format!(
+			"ext_{}_{}_version_1",
+			trait_name.to_string().to_snake_case(),
+			name,
+		),
+		Span::call_site(),
+	)
+}
+
+/// Returns the function arguments of the given `Signature`, minus any `self` arguments.
+pub fn get_function_arguments<'a>(sig: &'a Signature) -> impl Iterator<Item = PatType> + 'a {
+	sig.inputs
+		.iter()
+		.filter_map(|a| match a {
+			FnArg::Receiver(_) => None,
+			FnArg::Typed(pat_type) => Some(pat_type),
+		})
+		.enumerate()
+		.map(|(i, arg)| {
+			let mut res = arg.clone();
+			if let Pat::Wild(wild) = &*arg.pat {
+				let ident = Ident::new(
+					&format!("__runtime_interface_generated_{}_", i),
+					wild.span(),
+				);
+
+				res.pat = Box::new(parse_quote!( #ident ))
+			}
+
+			res
+		})
+}
+
+/// Returns the function argument names of the given `Signature`, minus any `self`.
+pub fn get_function_argument_names<'a>(sig: &'a Signature) -> impl Iterator<Item = Box<Pat>> + 'a {
+	get_function_arguments(sig).map(|pt| pt.pat)
+}
+
+/// Returns the function argument types of the given `Signature`, minus any `Self` type.
+pub fn get_function_argument_types<'a>(sig: &'a Signature) -> impl Iterator<Item = Box<Type>> + 'a {
+	get_function_arguments(sig).map(|pt| pt.ty)
+}
+
+/// Returns the function argument types, minus any `Self` type. If any of the arguments
+/// is a reference, the underlying type without the ref is returned.
+pub fn get_function_argument_types_without_ref<'a>(
+	sig: &'a Signature,
+) -> impl Iterator<Item = Box<Type>> + 'a {
+	get_function_arguments(sig)
+		.map(|pt| pt.ty)
+		.map(|ty| match *ty {
+			Type::Reference(type_ref) => type_ref.elem,
+			_ => ty,
+		})
+}
+
+/// Returns the function argument names and types, minus any `self`. If any of the arguments
+/// is a reference, the underlying type without the ref is returned.
+pub fn get_function_argument_names_and_types_without_ref<'a>(
+	sig: &'a Signature,
+) -> impl Iterator<Item = (Box<Pat>, Box<Type>)> + 'a {
+	get_function_arguments(sig)
+		.map(|pt| match *pt.ty {
+			Type::Reference(type_ref) => (pt.pat, type_ref.elem),
+			_ => (pt.pat, pt.ty),
+		})
+}
+
+/// Returns the `&`/`&mut` for all function argument types, minus the `self` arg. If a function
+/// argument is not a reference, `None` is returned.
+pub fn get_function_argument_types_ref_and_mut<'a>(
+	sig: &'a Signature,
+) -> impl Iterator<Item = Option<(token::And, Option<token::Mut>)>> + 'a {
+	get_function_arguments(sig)
+		.map(|pt| pt.ty)
+		.map(|ty| match *ty {
+			Type::Reference(type_ref) => Some((type_ref.and_token, type_ref.mutability)),
+			_ => None,
+		})
+}
+
+/// Returns an iterator over all trait methods for the given trait definition.
+pub fn get_trait_methods<'a>(trait_def: &'a ItemTrait) -> impl Iterator<Item = &'a TraitItemMethod> {
+	trait_def
+		.items
+		.iter()
+		.filter_map(|i| match i {
+			TraitItem::Method(ref method) => Some(method),
+			_ => None,
+		})
+}

substrate/primitives/runtime-interface/proc-macro/tests/ui.rs

@@ -0,0 +1,27 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::env;
+
+#[rustversion::attr(not(stable), ignore)]
+#[test]
+fn ui() {
+	// As trybuild is using `cargo check`, we don't need the real WASM binaries.
+	env::set_var("BUILD_DUMMY_WASM_BINARY", "1");
+
+	let t = trybuild::TestCases::new();
+	t.compile_fail("tests/ui/*.rs");
+}

substrate/primitives/runtime-interface/proc-macro/tests/ui/no_generic_parameters.rs

@@ -0,0 +1,8 @@
+use runtime_interface::runtime_interface;
+
+#[runtime_interface]
+trait Test<T> {
+	fn test<R>() {}
+}
+
+fn main() {}

substrate/primitives/runtime-interface/proc-macro/tests/ui/no_generic_parameters.stderr

@@ -0,0 +1,11 @@
+error: Generic parameters not supported.
+ --> $DIR/no_generic_parameters.rs:5:10
+  |
+5 |     fn test<R>() {}
+  |             ^
+
+error: Generic parameters not supported.
+ --> $DIR/no_generic_parameters.rs:4:12
+  |
+4 | trait Test<T> {
+  |            ^

substrate/primitives/runtime-interface/proc-macro/tests/ui/no_method_implementation.rs

@@ -0,0 +1,8 @@
+use runtime_interface::runtime_interface;
+
+#[runtime_interface]
+trait Test {
+	fn test();
+}
+
+fn main() {}

substrate/primitives/runtime-interface/proc-macro/tests/ui/no_method_implementation.stderr

@@ -0,0 +1,5 @@
+error: Methods need to have an implementation.
+ --> $DIR/no_method_implementation.rs:5:2
+  |
+5 |     fn test();
+  |     ^^

substrate/primitives/runtime-interface/proc-macro/tests/ui/pass_by_enum_with_struct.rs

@@ -0,0 +1,6 @@
+use runtime_interface::pass_by::PassByEnum;
+
+#[derive(PassByEnum)]
+struct Test;
+
+fn main() {}

substrate/primitives/runtime-interface/proc-macro/tests/ui/pass_by_enum_with_struct.stderr

@@ -0,0 +1,5 @@
+error: `PassByEnum` only supports enums as input type.
+ --> $DIR/pass_by_enum_with_struct.rs:3:10
+  |
+3 | #[derive(PassByEnum)]
+  |          ^^^^^^^^^^

substrate/primitives/runtime-interface/proc-macro/tests/ui/pass_by_enum_with_value_variant.rs

@@ -0,0 +1,8 @@
+use runtime_interface::pass_by::PassByEnum;
+
+#[derive(PassByEnum)]
+enum Test {
+	Var0(u32),
+}
+
+fn main() {}