Substrate runtime interface 2.0 (#4057)

* Adds first version of traits for generating the host functions

* First steps of the procedural macro

* Implements generation of the host extern functions

* Prefix ext host function with snake case trait name

* Implement host functions implementation on the host

* Change `HostFunctions` interface

* Implement `HostFunctions` for tuples

* Make `WasmExecutor` generic over the host functions

* Begin to add a test and make it compile

* Make the test succeed

* Add test to ensure that host functions are not found

* It's alive! Make the `set_storage` test work

* Add test for mutable references

* Code cleanup and documentation etc

* Add marker trait for types that should be passed as SCALE encoded

* Inherit the visibility from the trait and more improvements

* More impls and move them into their own file

* Code simplification by dropping one trait

* Give it a better name

* Implement traits for arrays

* Refactor code to support pass by codec/inner

* Docs

* Implement pass by inner for some crypto types and add a test

* Implement exchangeable function support

* Rewrite sr-io with as runtime interface

* Start reworking after master merge

* Adds `PassByCodec` derive

* Adds `PassByInner` derive

* Fix compilation errors

* More implementations

* Implement runtime interface traits for `str`

* Make `sr-io` compile again

* Fix more compilation errors

* More progress on getting stuff back to compile

* More compilation fixes

* Fix warnings

* Remove le conversions

* Add support for `wasm_only` interfaces

* Implement `Allocator` interface

* Improve error message

* Move `WasmAllocator` to `sr-io` and more clean ups

* Use correct function signature for wasm functions

* Store the host functions with the Wasm runtime

* Docs update

* Fix compilation after master merge

* Remove `sr-io/without_std`

* Make `srml-support` tests run again

* More compilation error fixes

* Use correct doc syntax

* Fix test-runtime

* Fix compilation

* Catch native panics when executing the wasm runtime

As with the native runtime, we now catch all native panics when we
execute the wasm runtime. The panics inside the wasm runtime were
already catched before by the wasm executor automatically, but any panic
in the host functions could bring down the node. The recent switch to
execute the native counterpart of the host function in `sr-io`, makes
this change required. The native `sr-io` functions just `panic` when
something is not provided or any other error occured.

* Fix compilation

* Don't panic in a panic

* Move `sr-sandbox` to new runtime interface

* Fixes tests after sandbox changes

* Make sure we detect invalid utf8

* Fixes after master merge

* Adds pass by enum strategy

* Fix wasmtime integration

* Some macro structure clean up

* Rework and test exchangebale host functions

* PassBy derive macros documentation

* Docs for `runtime_interface` macro

* Support wild card argument names

* Adds ui tests

* Make sure that we are backwards compatible to the old runtime interfaces

* Documentation

* Fixes after latest master merge

* Make `wasmtime` happy

* Make `full_crypto` work

* Make the new interface versionable

* Rename `Sanboxing` to `Sandbox`

* Don't finalize in test while importing

* Fix Performance regression

* Fix test

substrate/core/application-crypto/src/ed25519.rs

@@ -18,6 +18,8 @@
 
 use crate::{RuntimePublic, KeyTypeId};
 
+use rstd::vec::Vec;
+
 pub use primitives::ed25519::*;
 
 mod app {
@@ -29,8 +31,7 @@ mod app {
 	}
 }
 
-pub use app::Public as AppPublic;
-pub use app::Signature as AppSignature;
+pub use app::{Public as AppPublic, Signature as AppSignature};
 #[cfg(feature = "full_crypto")]
 pub use app::Pair as AppPair;
 
@@ -38,19 +39,19 @@ impl RuntimePublic for Public {
 	type Signature = Signature;
 
 	fn all(key_type: KeyTypeId) -> crate::Vec<Self> {
-		runtime_io::ed25519_public_keys(key_type)
+		runtime_io::crypto::ed25519_public_keys(key_type)
 	}
 
-	fn generate_pair(key_type: KeyTypeId, seed: Option<&str>) -> Self {
-		runtime_io::ed25519_generate(key_type, seed)
+	fn generate_pair(key_type: KeyTypeId, seed: Option<Vec<u8>>) -> Self {
+		runtime_io::crypto::ed25519_generate(key_type, seed)
 	}
 
 	fn sign<M: AsRef<[u8]>>(&self, key_type: KeyTypeId, msg: &M) -> Option<Self::Signature> {
-		runtime_io::ed25519_sign(key_type, self, msg.as_ref())
+		runtime_io::crypto::ed25519_sign(key_type, self, msg.as_ref())
 	}
 
 	fn verify<M: AsRef<[u8]>>(&self, msg: &M, signature: &Self::Signature) -> bool {
-		runtime_io::ed25519_verify(&signature, msg.as_ref(), self)
+		runtime_io::crypto::ed25519_verify(&signature, msg.as_ref(), self)
 	}
 }
 

substrate/core/application-crypto/src/lib.rs

@@ -286,7 +286,7 @@ macro_rules! app_crypto_public_common {
 				<$public as $crate::RuntimePublic>::all($key_type).into_iter().map(Self).collect()
 			}
 
-			fn generate_pair(seed: Option<&str>) -> Self {
+			fn generate_pair(seed: Option<$crate::Vec<u8>>) -> Self {
 				Self(<$public as $crate::RuntimePublic>::generate_pair($key_type, seed))
 			}
 

substrate/core/application-crypto/src/sr25519.rs

@@ -18,6 +18,8 @@
 
 use crate::{RuntimePublic, KeyTypeId};
 
+use rstd::vec::Vec;
+
 pub use primitives::sr25519::*;
 
 mod app {
@@ -29,8 +31,7 @@ mod app {
 	}
 }
 
-pub use app::Public as AppPublic;
-pub use app::Signature as AppSignature;
+pub use app::{Public as AppPublic, Signature as AppSignature};
 #[cfg(feature = "full_crypto")]
 pub use app::Pair as AppPair;
 
@@ -38,19 +39,19 @@ impl RuntimePublic for Public {
 	type Signature = Signature;
 
 	fn all(key_type: KeyTypeId) -> crate::Vec<Self> {
-		runtime_io::sr25519_public_keys(key_type)
+		runtime_io::crypto::sr25519_public_keys(key_type)
 	}
 
-	fn generate_pair(key_type: KeyTypeId, seed: Option<&str>) -> Self {
-		runtime_io::sr25519_generate(key_type, seed)
+	fn generate_pair(key_type: KeyTypeId, seed: Option<Vec<u8>>) -> Self {
+		runtime_io::crypto::sr25519_generate(key_type, seed)
 	}
 
 	fn sign<M: AsRef<[u8]>>(&self, key_type: KeyTypeId, msg: &M) -> Option<Self::Signature> {
-		runtime_io::sr25519_sign(key_type, self, msg.as_ref())
+		runtime_io::crypto::sr25519_sign(key_type, self, msg.as_ref())
 	}
 
 	fn verify<M: AsRef<[u8]>>(&self, msg: &M, signature: &Self::Signature) -> bool {
-		runtime_io::sr25519_verify(&signature, msg.as_ref(), self)
+		runtime_io::crypto::sr25519_verify(&signature, msg.as_ref(), self)
 	}
 }
 

substrate/core/application-crypto/src/traits.rs

@@ -19,7 +19,7 @@ use primitives::crypto::Pair;
 
 use codec::Codec;
 use primitives::crypto::{KeyTypeId, CryptoType, IsWrappedBy, Public};
-use rstd::fmt::Debug;
+use rstd::{fmt::Debug, vec::Vec};
 
 /// An application-specific key.
 pub trait AppKey: 'static + Send + Sync + Sized + CryptoType + Clone {
@@ -88,10 +88,13 @@ pub trait RuntimePublic: Sized {
 	/// Returns all public keys for the given key type in the keystore.
 	fn all(key_type: KeyTypeId) -> crate::Vec<Self>;
 
-	/// Generate a public/private pair for the given key type and store it in the keystore.
+	/// Generate a public/private pair for the given key type with an optional `seed` and
+	/// store it in the keystore.
+	///
+	/// The `seed` needs to be valid utf8.
 	///
 	/// Returns the generated public key.
-	fn generate_pair(key_type: KeyTypeId, seed: Option<&str>) -> Self;
+	fn generate_pair(key_type: KeyTypeId, seed: Option<Vec<u8>>) -> Self;
 
 	/// Sign the given message with the corresponding private key of this public key.
 	///
@@ -116,10 +119,12 @@ pub trait RuntimeAppPublic: Sized  {
 	/// Returns all public keys for this application in the keystore.
 	fn all() -> crate::Vec<Self>;
 
-	/// Generate a public/private pair and store it in the keystore.
+	/// Generate a public/private pair with an optional `seed` and store it in the keystore.
+	///
+	/// The `seed` needs to be valid utf8.
 	///
 	/// Returns the generated public key.
-	fn generate_pair(seed: Option<&str>) -> Self;
+	fn generate_pair(seed: Option<Vec<u8>>) -> Self;
 
 	/// Sign the given message with the corresponding private key of this public key.
 	///