Replace JSON metadata with custom metadata structures (#766)

* Move module metadata from json string to custom metadata

* Revisit the metadata structures

1. Move the structures into the metadata crate.
2. Switch to using Cow/MaybeOwnedArray to support Encode/Decode

* Adapt to new metadata structures

* Convert event json metadata to new metadata structures

* Convert storage json metadata to new metadata structures

* Convert runtime metadata from json to new metadata structs

* Implements new metadata structures in client and runtime

* Fixes errors on `no_std`

* Fixes errors after rebasing master

* Do not use `Cow` anymore in metadata

Also replace `String` with our own type definition `StringBuf`.
This fixes compilation on `no_std`.

* Wrap `RuntimeMetadata` in `RuntimeMetadataVersioned` to support versioning

* Move metadata into `srml` and make core unaware of the implementation

substrate/core/client/Cargo.toml

@@ -18,7 +18,6 @@ parity-codec = { version = "1.1" }
 substrate-executor = { path = "../executor" }
 substrate-primitives = { path = "../primitives" }
 sr-io = { path = "../sr-io" }
-substrate-metadata = { path = "../metadata" }
 sr-primitives = { path = "../sr-primitives" }
 substrate-state-machine = { path = "../state-machine" }
 substrate-keyring = { path = "../../core/keyring" }

substrate/core/client/src/client.rs

@@ -23,7 +23,6 @@ use primitives::AuthorityId;
 use runtime_primitives::{bft::Justification, generic::{BlockId, SignedBlock, Block as RuntimeBlock}};
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero, One, As, NumberFor};
 use runtime_primitives::BuildStorage;
-use substrate_metadata::JsonMetadataDecodable;
 use primitives::{Blake2Hasher, RlpCodec, H256};
 use primitives::storage::{StorageKey, StorageData};
 use primitives::storage::well_known_keys;
@@ -254,26 +253,9 @@ impl<B, E, Block> Client<B, E, Block> where
 		&self.executor
 	}
 
-	/// Returns the runtime metadata as JSON.
-	pub fn json_metadata(&self, id: &BlockId<Block>) -> error::Result<String> {
-		self.executor.call(id, "json_metadata",&[])
-			.and_then(|r| Vec::<JsonMetadataDecodable>::decode(&mut &r.return_data[..])
-					  .ok_or("JSON Metadata decoding failed".into()))
-			.and_then(|metadata| {
-				let mut json = metadata.into_iter().enumerate().fold(String::from("{"),
-					|mut json, (i, m)| {
-						if i > 0 {
-							json.push_str(",");
-						}
-						let (mtype, val) = m.into_json_string();
-						json.push_str(&format!(r#" "{}": {}"#, mtype, val));
-						json
-					}
-				);
-				json.push_str(" }");
-
-				Ok(json)
-			})
+	/// Returns the runtime metadata.
+	pub fn metadata(&self, id: &BlockId<Block>) -> error::Result<Vec<u8>> {
+		self.executor.call(id, "metadata",&[]).map(|v| v.return_data)
 	}
 
 	/// Reads storage value at a given block + key, returning read proof.
@@ -772,32 +754,4 @@ mod tests {
 		assert!(client.state_at(&BlockId::Number(1)).unwrap() != client.state_at(&BlockId::Number(0)).unwrap());
 		assert_eq!(client.body(&BlockId::Number(1)).unwrap().unwrap().len(), 1)
 	}
-
-	#[test]
-	fn json_metadata() {
-		let client = test_client::new();
-
-		let mut builder = client.new_block().unwrap();
-
-		builder.push_transfer(Transfer {
-			from: Keyring::Alice.to_raw_public().into(),
-			to: Keyring::Ferdie.to_raw_public().into(),
-			amount: 42,
-			nonce: 0,
-		}).unwrap();
-
-		assert!(builder.push_transfer(Transfer {
-			from: Keyring::Eve.to_raw_public().into(),
-			to: Keyring::Alice.to_raw_public().into(),
-			amount: 42,
-			nonce: 0,
-		}).is_err());
-
-		client.justify_and_import(BlockOrigin::Own, builder.bake().unwrap()).unwrap();
-
-		assert_eq!(
-			client.json_metadata(&BlockId::Number(1)).unwrap(),
-			r#"{ "events": { "name": "Test", "events": { "event": hallo } } }"#
-		);
-	}
 }

substrate/core/client/src/lib.rs

@@ -21,7 +21,6 @@
 
 extern crate substrate_bft as bft;
 extern crate parity_codec as codec;
-extern crate substrate_metadata;
 extern crate substrate_primitives as primitives;
 extern crate sr_io as runtime_io;
 extern crate sr_primitives as runtime_primitives;

substrate/core/metadata/Cargo.toml

@@ -1,13 +0,0 @@
-[package]
-name = "substrate-metadata"
-version = "0.1.0"
-authors = ["Parity Technologies <admin@parity.io>"]
-
-[dependencies]
-parity-codec = { version = "1.1", default_features = false }
-
-[features]
-default = ["std"]
-std = [
-	"parity-codec/std"
-]

substrate/core/metadata/README.adoc

@@ -1,13 +0,0 @@
-
-= Substrate BFT
-
-.Summary
-[source, toml]
-----
-include::Cargo.toml[lines=2..5]
-----
-
-.Description
-----
-include::src/lib.rs[tag=description]
-----

substrate/core/metadata/src/lib.rs

@@ -1,190 +0,0 @@
-// Copyright 2017 Parity Technologies (UK) Ltd.
-// This file is part of Substrate.
-
-// Substrate is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Substrate is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
-
-//! Decodable variant of the JsonMetadata.
-//!
-//! This really doesn't belong here, but is necessary for the moment. In the future
-//! it should be removed entirely to an external module for shimming on to the
-//! codec-encoded metadata.
-
-#![cfg_attr(not(feature = "std"), no_std)]
-
-extern crate parity_codec as codec;
-
-use codec::{Encode, Output};
-#[cfg(feature = "std")]
-use codec::{Decode, Input};
-
-/// The metadata of a runtime encoded as JSON.
-#[derive(Eq)]
-#[cfg_attr(feature = "std", derive(Debug))]
-pub enum JsonMetadata {
-	Events { name: &'static str, events: &'static [(&'static str, fn() -> &'static str)] },
-	Module { module: &'static str, prefix: &'static str },
-	ModuleWithStorage { module: &'static str, prefix: &'static str, storage: &'static str }
-}
-
-impl Encode for JsonMetadata {
-	fn encode_to<W: Output>(&self, dest: &mut W) {
-		match self {
-			JsonMetadata::Events { name, events } => {
-				0i8.encode_to(dest);
-				name.encode_to(dest);
-				events.iter().fold(0u32, |count, _| count + 1).encode_to(dest);
-				events
-					.iter()
-					.map(|(module, data)| (module, data()))
-					.for_each(|val| val.encode_to(dest));
-			},
-			JsonMetadata::Module { module, prefix } => {
-				1i8.encode_to(dest);
-				prefix.encode_to(dest);
-				module.encode_to(dest);
-			},
-			JsonMetadata::ModuleWithStorage { module, prefix, storage } => {
-				2i8.encode_to(dest);
-				prefix.encode_to(dest);
-				module.encode_to(dest);
-				storage.encode_to(dest);
-			}
-		}
-	}
-}
-
-impl PartialEq<JsonMetadata> for JsonMetadata {
-	fn eq(&self, other: &JsonMetadata) -> bool {
-		match (self, other) {
-			(
-				JsonMetadata::Events { name: lname, events: left },
-				JsonMetadata::Events { name: rname, events: right }
-			) => {
-				lname == rname && left.iter().zip(right.iter()).fold(true, |res, (l, r)| {
-					res && l.0 == r.0 && l.1() == r.1()
-				})
-			},
-			(
-				JsonMetadata::Module { prefix: lpre, module: lmod },
-				JsonMetadata::Module { prefix: rpre, module: rmod }
-			) => {
-				lpre == rpre && lmod == rmod
-			},
-			(
-				JsonMetadata::ModuleWithStorage { prefix: lpre, module: lmod, storage: lstore },
-				JsonMetadata::ModuleWithStorage { prefix: rpre, module: rmod, storage: rstore }
-			) => {
-				lpre == rpre && lmod == rmod && lstore == rstore
-			},
-			_ => false,
-		}
-    }
-}
-
-/// Utility struct for making `JsonMetadata` decodeable.
-#[derive(Eq, PartialEq, Debug)]
-#[cfg(feature = "std")]
-pub enum JsonMetadataDecodable {
-	Events { name: String, events: Vec<(String, String)> },
-	Module { module: String, prefix: String },
-	ModuleWithStorage { module: String, prefix: String, storage: String }
-}
-
-#[cfg(feature = "std")]
-impl JsonMetadataDecodable {
-	/// Returns the instance as JSON string.
-	/// The first value of the tuple is the name of the metadata type and the second in the JSON string.
-	pub fn into_json_string(self) -> (&'static str, String) {
-		match self {
-			JsonMetadataDecodable::Events { name, events } => {
-				(
-					"events",
-					format!(
-						r#"{{ "name": "{}", "events": {{ {} }} }}"#, name,
-						events.iter().enumerate()
-							.fold(String::from(""), |mut json, (i, (name, data))| {
-								if i > 0 {
-									json.push_str(", ");
-								}
-								json.push_str(&format!(r#""{}": {}"#, name, data));
-								json
-							})
-					)
-				)
-			},
-			JsonMetadataDecodable::Module { prefix, module } => {
-				("module", format!(r#"{{ "prefix": "{}", "module": {} }}"#, prefix, module))
-			},
-			JsonMetadataDecodable::ModuleWithStorage { prefix, module, storage } => {
-				(
-					"moduleWithStorage",
-					format!(
-						r#"{{ "prefix": "{}", "module": {}, "storage": {} }}"#,
-						prefix, module, storage
-					)
-				)
-			}
-		}
-	}
-}
-
-#[cfg(feature = "std")]
-impl Decode for JsonMetadataDecodable {
-	fn decode<I: Input>(input: &mut I) -> Option<Self> {
-		i8::decode(input).and_then(|variant| {
-			match variant {
-				0 => String::decode(input)
-						.and_then(|name| Vec::<(String, String)>::decode(input).map(|events| (name, events)))
-						.and_then(|(name, events)| Some(JsonMetadataDecodable::Events { name, events })),
-				1 => String::decode(input)
-						.and_then(|prefix| String::decode(input).map(|v| (prefix, v)))
-						.and_then(|(prefix, module)| Some(JsonMetadataDecodable::Module { prefix, module })),
-				2 => String::decode(input)
-						.and_then(|prefix| String::decode(input).map(|v| (prefix, v)))
-						.and_then(|(prefix, module)| String::decode(input).map(|v| (prefix, module, v)))
-						.and_then(|(prefix, module, storage)| Some(JsonMetadataDecodable::ModuleWithStorage { prefix, module, storage })),
-				_ => None,
-			}
-		})
-	}
-}
-
-#[cfg(feature = "std")]
-impl PartialEq<JsonMetadata> for JsonMetadataDecodable {
-	fn eq(&self, other: &JsonMetadata) -> bool {
-		match (self, other) {
-			(
-				JsonMetadataDecodable::Events { name: lname, events: left },
-				JsonMetadata::Events { name: rname, events: right }
-			) => {
-				lname == rname && left.iter().zip(right.iter()).fold(true, |res, (l, r)| {
-					res && l.0 == r.0 && l.1 == r.1()
-				})
-			},
-			(
-				JsonMetadataDecodable::Module { prefix: lpre, module: lmod },
-				JsonMetadata::Module { prefix: rpre, module: rmod }
-			) => {
-				lpre == rpre && lmod == rmod
-			},
-			(
-				JsonMetadataDecodable::ModuleWithStorage { prefix: lpre, module: lmod, storage: lstore },
-				JsonMetadata::ModuleWithStorage { prefix: rpre, module: rmod, storage: rstore }
-			) => {
-				lpre == rpre && lmod == rmod && lstore == rstore
-			},
-			_ => false,
-		}
-    }
-}

substrate/core/rpc/src/state/mod.rs

@@ -145,8 +145,8 @@ impl<B, E, Block> StateApi<Block::Hash> for State<B, E, Block> where
 
 	fn metadata(&self, block: Trailing<Block::Hash>) -> Result<serde_json::Value> {
 		let block = self.unwrap_or_best(block)?;
-		let metadata = self.client.json_metadata(&BlockId::Hash(block))?;
-		serde_json::from_str(&metadata).map_err(Into::into)
+		let metadata = self.client.metadata(&BlockId::Hash(block))?;
+		serde_json::to_value(metadata).map_err(Into::into)
 	}
 
 	fn query_storage(&self, keys: Vec<StorageKey>, from: Block::Hash, to: Trailing<Block::Hash>) -> Result<Vec<StorageChangeSet<Block::Hash>>> {

substrate/core/test-runtime/src/lib.rs

@@ -133,21 +133,10 @@ pub fn run_tests(mut input: &[u8]) -> Vec<u8> {
 	[stxs.len() as u8].encode()
 }
 
-fn test_event_json() -> &'static str {
-	"hallo"
-}
-
 pub mod api {
 	use system;
 	impl_stubs!(
 		version => |()| super::version(),
-		json_metadata => |()| {
-			let mut vec = ::runtime_support::metadata::Vec::new();
-			vec.push(::runtime_support::metadata::JsonMetadata::Events {
-				name: "Test", events: &[ ("event", super::test_event_json) ]
-			});
-			vec
-		},
 		authorities => |()| system::authorities(),
 		initialise_block => |header| system::initialise_block(header),
 		execute_block => |block| system::execute_block(block),

substrate/core/test-runtime/wasm/Cargo.lock

@@ -588,6 +588,9 @@ name = "substrate-metadata"
 version = "0.1.0"
 dependencies = [
  "parity-codec 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "parity-codec-derive 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_derive 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]