Substrate runtime interface 2.0 (#4057)

* Adds first version of traits for generating the host functions

* First steps of the procedural macro

* Implements generation of the host extern functions

* Prefix ext host function with snake case trait name

* Implement host functions implementation on the host

* Change `HostFunctions` interface

* Implement `HostFunctions` for tuples

* Make `WasmExecutor` generic over the host functions

* Begin to add a test and make it compile

* Make the test succeed

* Add test to ensure that host functions are not found

* It's alive! Make the `set_storage` test work

* Add test for mutable references

* Code cleanup and documentation etc

* Add marker trait for types that should be passed as SCALE encoded

* Inherit the visibility from the trait and more improvements

* More impls and move them into their own file

* Code simplification by dropping one trait

* Give it a better name

* Implement traits for arrays

* Refactor code to support pass by codec/inner

* Docs

* Implement pass by inner for some crypto types and add a test

* Implement exchangeable function support

* Rewrite sr-io with as runtime interface

* Start reworking after master merge

* Adds `PassByCodec` derive

* Adds `PassByInner` derive

* Fix compilation errors

* More implementations

* Implement runtime interface traits for `str`

* Make `sr-io` compile again

* Fix more compilation errors

* More progress on getting stuff back to compile

* More compilation fixes

* Fix warnings

* Remove le conversions

* Add support for `wasm_only` interfaces

* Implement `Allocator` interface

* Improve error message

* Move `WasmAllocator` to `sr-io` and more clean ups

* Use correct function signature for wasm functions

* Store the host functions with the Wasm runtime

* Docs update

* Fix compilation after master merge

* Remove `sr-io/without_std`

* Make `srml-support` tests run again

* More compilation error fixes

* Use correct doc syntax

* Fix test-runtime

* Fix compilation

* Catch native panics when executing the wasm runtime

As with the native runtime, we now catch all native panics when we
execute the wasm runtime. The panics inside the wasm runtime were
already catched before by the wasm executor automatically, but any panic
in the host functions could bring down the node. The recent switch to
execute the native counterpart of the host function in `sr-io`, makes
this change required. The native `sr-io` functions just `panic` when
something is not provided or any other error occured.

* Fix compilation

* Don't panic in a panic

* Move `sr-sandbox` to new runtime interface

* Fixes tests after sandbox changes

* Make sure we detect invalid utf8

* Fixes after master merge

* Adds pass by enum strategy

* Fix wasmtime integration

* Some macro structure clean up

* Rework and test exchangebale host functions

* PassBy derive macros documentation

* Docs for `runtime_interface` macro

* Support wild card argument names

* Adds ui tests

* Make sure that we are backwards compatible to the old runtime interfaces

* Documentation

* Fixes after latest master merge

* Make `wasmtime` happy

* Make `full_crypto` work

* Make the new interface versionable

* Rename `Sanboxing` to `Sandbox`

* Don't finalize in test while importing

* Fix Performance regression

* Fix test

substrate/core/primitives/Cargo.toml

@@ -36,6 +36,7 @@ tiny-keccak = { version = "1.5.0", optional = true }
 substrate-debug-derive = { version = "2.0.0", path = "./debug-derive" }
 externalities = { package = "substrate-externalities", path = "../externalities", optional = true }
 primitives-storage = { package = "substrate-primitives-storage", path = "storage", default-features = false }
+runtime-interface = { package = "substrate-runtime-interface", path = "../runtime-interface", default-features = false }
 
 [dev-dependencies]
 substrate-serializer = { path = "../serializer" }
@@ -90,10 +91,11 @@ std = [
 	"substrate-debug-derive/std",
 	"externalities",
 	"primitives-storage/std",
+	"runtime-interface/std",
 ]
 
-# This feature enables all crypto primitives for `no_std` builds like microcontrollers 
-# or Intel SGX. 
+# This feature enables all crypto primitives for `no_std` builds like microcontrollers
+# or Intel SGX.
 # For the regular wasm runtime builds this should not be used.
 full_crypto = [
 	"ed25519-dalek",
@@ -102,5 +104,6 @@ full_crypto = [
 	"libsecp256k1",
 	"hex",
 	"sha2",
-	"twox-hash"
+	"twox-hash",
+	"runtime-interface/disable_target_static_assertions",
 ]

substrate/core/primitives/src/crypto.rs

@@ -35,6 +35,7 @@ use base58::{FromBase58, ToBase58};
 use zeroize::Zeroize;
 #[doc(hidden)]
 pub use rstd::ops::Deref;
+use runtime_interface::pass_by::PassByInner;
 
 /// The root phrase for our publicly known keys.
 pub const DEV_PHRASE: &str = "bottom drive obey lake curtain smoke basket hold race lonely fit walk";
@@ -866,8 +867,10 @@ pub trait CryptoType {
 ///
 /// Values whose first character is `_` are reserved for private use and won't conflict with any
 /// public modules.
-#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Encode, Decode)]
-#[derive(crate::RuntimeDebug)]
+#[derive(
+	Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Encode, Decode, PassByInner,
+	crate::RuntimeDebug
+)]
 pub struct KeyTypeId(pub [u8; 4]);
 
 impl From<u32> for KeyTypeId {

substrate/core/primitives/src/ecdsa.rs

@@ -18,6 +18,7 @@
 //! Simple ECDSA API.
 // end::description[]
 
+#[cfg(feature = "full_crypto")]
 use rstd::vec::Vec;
 
 use rstd::cmp::Ordering;

substrate/core/primitives/src/ed25519.rs

@@ -18,6 +18,7 @@
 //! Simple Ed25519 API.
 // end::description[]
 
+#[cfg(feature = "full_crypto")]
 use rstd::vec::Vec;
 
 use crate::{hash::H256, hash::H512};
@@ -36,6 +37,7 @@ use crate::crypto::Ss58Codec;
 #[cfg(feature = "std")]
 use serde::{de, Serializer, Serialize, Deserializer, Deserialize};
 use crate::{crypto::{Public as TraitPublic, UncheckedFrom, CryptoType, Derive}};
+use runtime_interface::pass_by::PassByInner;
 
 /// A secret seed. It's not called a "secret key" because ring doesn't expose the secret keys
 /// of the key pair (yeah, dumb); as such we're forced to remember the seed manually if we
@@ -45,7 +47,7 @@ type Seed = [u8; 32];
 
 /// A public key.
 #[cfg_attr(feature = "full_crypto", derive(Hash))]
-#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Encode, Decode, Default)]
+#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Encode, Decode, Default, PassByInner)]
 pub struct Public(pub [u8; 32]);
 
 /// A key pair.
@@ -162,7 +164,7 @@ impl<'de> Deserialize<'de> for Public {
 }
 
 /// A signature (a 512-bit value).
-#[derive(Encode, Decode)]
+#[derive(Encode, Decode, PassByInner)]
 pub struct Signature(pub [u8; 64]);
 
 impl rstd::convert::TryFrom<&[u8]> for Signature {

substrate/core/primitives/src/lib.rs

@@ -236,7 +236,7 @@ pub trait TypeId {
 /// A log level matching the one from `log` crate.
 ///
 /// Used internally by `runtime_io::log` method.
-#[repr(u32)]
+#[derive(Encode, Decode, runtime_interface::pass_by::PassByEnum, Copy, Clone)]
 pub enum LogLevel {
 	/// `Error` log level.
 	Error = 1,

substrate/core/primitives/src/offchain.rs

@@ -19,11 +19,12 @@
 use codec::{Encode, Decode};
 use rstd::{prelude::{Vec, Box}, convert::TryFrom};
 use crate::RuntimeDebug;
+use runtime_interface::pass_by::{PassByCodec, PassByInner, PassByEnum};
 
 pub use crate::crypto::KeyTypeId;
 
 /// A type of supported crypto.
-#[derive(Clone, Copy, PartialEq, Eq, Encode, Decode, RuntimeDebug)]
+#[derive(Clone, Copy, PartialEq, Eq, Encode, Decode, RuntimeDebug, PassByEnum)]
 #[repr(C)]
 pub enum StorageKind {
 	/// Persistent storage is non-revertible and not fork-aware. It means that any value
@@ -59,7 +60,7 @@ impl From<StorageKind> for u32 {
 }
 
 /// Opaque type for offchain http requests.
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, RuntimeDebug)]
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, RuntimeDebug, Encode, Decode, PassByInner)]
 #[cfg_attr(feature = "std", derive(Hash))]
 pub struct HttpRequestId(pub u16);
 
@@ -70,7 +71,7 @@ impl From<HttpRequestId> for u32 {
 }
 
 /// An error enum returned by some http methods.
-#[derive(Clone, Copy, PartialEq, Eq, RuntimeDebug)]
+#[derive(Clone, Copy, PartialEq, Eq, RuntimeDebug, Encode, Decode, PassByEnum)]
 #[repr(C)]
 pub enum HttpError {
 	/// The requested action couldn't been completed within a deadline.
@@ -101,7 +102,7 @@ impl From<HttpError> for u32 {
 }
 
 /// Status of the HTTP request
-#[derive(Clone, Copy, PartialEq, Eq, RuntimeDebug)]
+#[derive(Clone, Copy, PartialEq, Eq, RuntimeDebug, Encode, Decode, PassByCodec)]
 pub enum HttpRequestStatus {
 	/// Deadline was reached while we waited for this request to finish.
 	///
@@ -147,7 +148,7 @@ impl TryFrom<u32> for HttpRequestStatus {
 
 /// A blob to hold information about the local node's network state
 /// without committing to its format.
-#[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug)]
+#[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug, PassByCodec)]
 pub struct OpaqueNetworkState {
 	/// PeerId of the local node.
 	pub peer_id: OpaquePeerId,
@@ -156,7 +157,7 @@ pub struct OpaqueNetworkState {
 }
 
 /// Simple blob to hold a `PeerId` without committing to its format.
-#[derive(Default, Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug)]
+#[derive(Default, Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug, PassByInner)]
 pub struct OpaquePeerId(pub Vec<u8>);
 
 impl OpaquePeerId {
@@ -167,7 +168,7 @@ impl OpaquePeerId {
 }
 
 /// Simple blob to hold a `Multiaddr` without committing to its format.
-#[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug)]
+#[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug, PassByInner)]
 pub struct OpaqueMultiaddr(pub Vec<u8>);
 
 impl OpaqueMultiaddr {
@@ -178,11 +179,11 @@ impl OpaqueMultiaddr {
 }
 
 /// Opaque timestamp type
-#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default, RuntimeDebug)]
+#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default, RuntimeDebug, PassByInner, Encode, Decode)]
 pub struct Timestamp(u64);
 
 /// Duration type
-#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default, RuntimeDebug)]
+#[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Default, RuntimeDebug, PassByInner, Encode, Decode)]
 pub struct Duration(u64);
 
 impl Duration {

substrate/core/primitives/src/sr25519.rs

@@ -20,6 +20,7 @@
 //! Note: `CHAIN_CODE_LENGTH` must be equal to `crate::crypto::JUNCTION_ID_LEN`
 //! for this to work.
 // end::description[]
+#[cfg(feature = "full_crypto")]
 use rstd::vec::Vec;
 #[cfg(feature = "full_crypto")]
 use schnorrkel::{signing_context, ExpansionMode, Keypair, SecretKey, MiniSecretKey, PublicKey,
@@ -44,6 +45,7 @@ use codec::{Encode, Decode};
 use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
 #[cfg(feature = "full_crypto")]
 use schnorrkel::keys::{MINI_SECRET_KEY_LENGTH, SECRET_KEY_LENGTH};
+use runtime_interface::pass_by::PassByInner;
 
 // signing context
 #[cfg(feature = "full_crypto")]
@@ -51,7 +53,7 @@ const SIGNING_CTX: &[u8] = b"substrate";
 
 /// An Schnorrkel/Ristretto x25519 ("sr25519") public key.
 #[cfg_attr(feature = "full_crypto", derive(Hash))]
-#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Encode, Decode, Default)]
+#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Encode, Decode, Default, PassByInner)]
 pub struct Public(pub [u8; 32]);
 
 /// An Schnorrkel/Ristretto x25519 ("sr25519") key pair.
@@ -163,7 +165,7 @@ impl<'de> Deserialize<'de> for Public {
 /// An Schnorrkel/Ristretto x25519 ("sr25519") signature.
 ///
 /// Instead of importing it for the local module, alias it to be available as a public type
-#[derive(Encode, Decode)]
+#[derive(Encode, Decode, PassByInner)]
 pub struct Signature(pub [u8; 64]);
 
 impl rstd::convert::TryFrom<&[u8]> for Signature {

substrate/core/primitives/storage/src/lib.rs

@@ -40,6 +40,14 @@ pub struct StorageData(
 	pub Vec<u8>,
 );
 
+/// A set of key value pairs for storage.
+#[cfg(feature = "std")]
+pub type StorageOverlay = std::collections::HashMap<Vec<u8>, Vec<u8>>;
+
+/// A set of key value pairs for children storage;
+#[cfg(feature = "std")]
+pub type ChildrenStorageOverlay = std::collections::HashMap<Vec<u8>, StorageOverlay>;
+
 /// Storage change set
 #[derive(RuntimeDebug)]
 #[cfg_attr(feature = "std", derive(Serialize, Deserialize, PartialEq, Eq))]