decl_storage as a proc_macro (#1176)

* `decl_storage` parsing of the macro (TODO change tool crate structure)

* Start formatting, for now use inner macro.
Still missing optional formating last part (genesis ...).

* Calling extra genesis macro

* decl_storage lines parsing.

* genesis macro as quote (need some cleaning reorg)

* dirty $crate substitute

* proc crate reorg.

* PR impl : skip usage of phantom data, it only applies in test and
council (others required it).

* Remaining macro of decl_storage, warning stringify behave sometime
oddly.

* Formatting code and some cleaning.

* Include line parsing to main struct (cannot use existing macro anymore).

* Remove genesis phantom data when there is already a field with type
parameter.

* Revert wasm files

* Remove old version of `decl_storage`.

* Fix false positive for phantom trait (additional type check on config
build).

* slight changes:
- return token errors instead of panic
- do not use useless intermediate vec

* Update srml/support/procedural/tools/derive/src/lib.rs

remove indent

Co-Authored-By: cheme <emericchevalier.pro@gmail.com>

* Switch iterations to fold, remove unused import.

substrate/srml/support/procedural/tools/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "srml-support-procedural-tools"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+
+[dependencies]
+srml-support-procedural-tools-derive = { path = "./derive" }
+proc-macro2 = "0.4"
+quote = { version = "0.6" }
+syn = { version = "0.15", features = ["full"] }

substrate/srml/support/procedural/tools/derive/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "srml-support-procedural-tools-derive"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+
+[lib]
+proc-macro = true
+
+[dependencies]
+proc-macro2 = "0.4.24"
+quote = { version = "0.6.10", features = ["proc-macro"] }
+syn = { version = "0.15.21", features = ["proc-macro" ,"full", "extra-traits", "parsing"] }

substrate/srml/support/procedural/tools/derive/src/lib.rs

@@ -0,0 +1,229 @@
+// Copyright 2017-2018 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+// tag::description[]
+//! Use to derive parsing for parsing struct.
+// end::description[]
+
+
+#![recursion_limit = "128"]
+
+#[macro_use]
+extern crate syn;
+
+#[macro_use]
+extern crate quote;
+
+extern crate proc_macro;
+extern crate proc_macro2;
+
+use proc_macro::TokenStream;
+use proc_macro2::Span;
+
+
+pub(crate) fn fields_idents(
+	fields: impl Iterator<Item = syn::Field>,
+) -> impl Iterator<Item = proc_macro2::TokenStream> {
+	fields.enumerate().map(|(ix, field)| {
+		field.ident.clone().map(|i| quote!{#i}).unwrap_or_else(|| {
+			let f_ix: syn::Ident = syn::Ident::new(&format!("f_{}", ix), Span::call_site());
+			quote!( #f_ix )
+		})
+	})
+}
+
+pub(crate) fn fields_access(
+	fields: impl Iterator<Item = syn::Field>,
+) -> impl Iterator<Item = proc_macro2::TokenStream> {
+	fields.enumerate().map(|(ix, field)| {
+		field.ident.clone().map(|i| quote!( #i )).unwrap_or_else(|| {
+			let f_ix: syn::Index = syn::Index {
+				index: ix as u32,
+				span: Span::call_site(),
+			};
+			quote!( #f_ix )
+		})
+	})
+}
+
+/// self defined parsing struct (use where clause on struct for it: not meant for good struct
+/// design but fast parse impl).
+#[proc_macro_derive(ParseStruct)]
+pub fn derive_parse_struct(input: TokenStream) -> TokenStream {
+	let syn::ItemStruct {
+		ident,
+		generics,
+		fields,
+		..
+	} = parse_macro_input!(input as syn::ItemStruct);
+	let field_names = {
+		let name = fields_idents(fields.iter().map(Clone::clone));
+		quote!{
+			#(
+				#name,
+			)*
+		}
+	};
+	let field = fields_idents(fields.iter().map(Clone::clone));
+	let tokens = quote! {
+		impl #generics syn::parse::Parse for #ident #generics {
+			fn parse(input: syn::parse::ParseStream) -> syn::parse::Result<Self> {
+				#(
+					let #field = input.parse()?;
+				)*
+				Ok(Self {
+					#field_names
+				})
+			}
+		}
+	};
+	tokens.into()
+}
+
+#[proc_macro_derive(ToTokensStruct)]
+pub fn derive_totokens_struct(input: TokenStream) -> TokenStream {
+	let syn::ItemStruct {
+		ident,
+		generics,
+		fields,
+		..
+	} = parse_macro_input!(input as syn::ItemStruct);
+	let fields = fields_access(fields.iter().map(Clone::clone));
+	let tokens = quote! {
+
+		impl #generics quote::ToTokens for #ident #generics {
+			fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) {
+				#(
+					self.#fields.to_tokens(tokens);
+				)*
+			}
+		}
+
+	};
+	tokens.into()
+}
+
+
+/// self defined parsing enum, variant are tested in order of definition.
+/// Empty variant allways true.
+/// Please use carefully, this will fully parse successfull variant twice.
+#[proc_macro_derive(ParseEnum)]
+pub fn derive_parse_enum(input: TokenStream) -> TokenStream {
+
+	let syn::ItemEnum {
+		ident,
+		generics,
+		variants,
+		..
+	} = parse_macro_input!(input as syn::ItemEnum);
+	let variants = variants.iter().map(|v| {
+		let variant_ident = v.ident.clone();
+		let fields_build = if v.fields.iter().count() > 0 {
+			let fields_id = fields_idents(v.fields.iter().map(Clone::clone));
+			quote!( (#(#fields_id), *) )
+		} else {
+			quote!()
+		};
+
+		let fields_procs = fields_idents(v.fields.iter().map(Clone::clone))
+			.map(|fident| {
+				quote!{
+					let mut #fident = match fork.parse() {
+						Ok(r) => r,
+						Err(_e) => break,
+					};
+				}
+			});
+		let fields_procs_again = fields_idents(v.fields.iter().map(Clone::clone))
+			.map(|fident| {
+				quote!{
+					#fident = input.parse().expect("was parsed just before");
+				}
+			});
+
+		// double parse to update input cursor position
+		// next syn crate version should be checked for a way
+		// to copy position/state from a fork
+		quote!{
+			let mut fork = input.fork();
+			loop {
+				#(#fields_procs)*
+				#(#fields_procs_again)*
+				return Ok(#ident::#variant_ident#fields_build);
+			}
+		}
+	});
+
+	let tokens = quote! {
+		impl #generics syn::parse::Parse for #ident #generics {
+			fn parse(input: syn::parse::ParseStream) -> syn::parse::Result<Self> {
+				#(
+					#variants
+				)*
+				// no early return from any variants
+				Err(
+					syn::parse::Error::new(
+						proc_macro2::Span::call_site(),
+						"derived enum no matching variants"
+					)
+				)
+			}
+		}
+
+	};
+	tokens.into()
+}
+
+/// only output field (empty field act as a None)
+#[proc_macro_derive(ToTokensEnum)]
+pub fn derive_totokens_enum(input: TokenStream) -> TokenStream {
+	let syn::ItemEnum {
+		ident,
+		generics,
+		variants,
+		..
+	} = parse_macro_input!(input as syn::ItemEnum);
+	let variants = variants.iter().map(|v| {
+		let v_ident = v.ident.clone();
+		let fields_build = if v.fields.iter().count() > 0 {
+			let fields_id = fields_idents(v.fields.iter().map(Clone::clone));
+			quote!( (#(#fields_id), *) )
+		} else {
+			quote!()
+		};
+		let field = fields_idents(v.fields.iter().map(Clone::clone));
+		quote! {
+			#ident::#v_ident#fields_build => {
+				#(
+					#field.to_tokens(tokens);
+				)*
+			},
+		}
+	});
+	let tokens = quote! {
+		impl #generics quote::ToTokens for #ident #generics {
+			fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) {
+				match self {
+					#(
+						#variants
+					)*
+				}
+			}
+		}
+	};
+
+	tokens.into()
+}

substrate/srml/support/procedural/tools/src/lib.rs

@@ -0,0 +1,94 @@
+// Copyright 2017-2018 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+// tag::description[]
+//! Proc macro helpers for procedural macros
+// end::description[]
+
+extern crate syn;
+#[macro_use]
+extern crate quote;
+extern crate proc_macro2;
+
+extern crate proc_macro;
+
+#[macro_use] extern crate srml_support_procedural_tools_derive;
+
+// reexport proc macros
+pub use srml_support_procedural_tools_derive::*;
+
+pub mod syn_ext;
+
+#[macro_export]
+macro_rules! custom_keyword_impl {
+	($name:ident, $keyident:expr, $keydisp:expr) => {
+
+		impl CustomKeyword for $name {
+			fn ident() -> &'static str { $keyident }
+			fn display() -> &'static str { $keydisp }
+		}
+
+	}
+}
+
+#[macro_export]
+macro_rules! custom_keyword {
+	($name:ident, $keyident:expr, $keydisp:expr) => {
+
+		#[derive(Debug)]
+		struct $name;
+
+		custom_keyword_impl!($name, $keyident, $keydisp);
+
+	}
+}
+
+
+// TODO following functions are copied from sr-api-macros : do a merge to get a unique procedural
+// macro tooling crate (this crate path does not look good for it)
+
+use proc_macro2::{TokenStream, Span};
+use syn::Ident;
+
+fn generate_hidden_includes_mod_name(unique_id: &str) -> Ident {
+	Ident::new(&format!("sr_api_hidden_includes_{}", unique_id), Span::call_site())
+}
+
+/// Generates the access to the `subtrate_client` crate.
+pub fn generate_crate_access(unique_id: &str, def_crate: &str) -> TokenStream {
+	if ::std::env::var("CARGO_PKG_NAME").unwrap() == def_crate {
+		quote!( crate )
+	} else {
+		let mod_name = generate_hidden_includes_mod_name(unique_id);
+		quote!( self::#mod_name::hidden_include )
+	}.into()
+}
+
+/// Generates the hidden includes that are required to make the macro independent from its scope.
+pub fn generate_hidden_includes(unique_id: &str, def_crate: &str, crate_id: &str) -> TokenStream {
+	let crate_id = Ident::new(crate_id, Span::call_site());
+	if ::std::env::var("CARGO_PKG_NAME").unwrap() == def_crate {
+		TokenStream::new()
+	} else {
+		let mod_name = generate_hidden_includes_mod_name(unique_id);
+		quote!(
+			#[doc(hidden)]
+			mod #mod_name {
+				pub extern crate #crate_id as hidden_include;
+			}
+		)
+	}.into()
+}

substrate/srml/support/procedural/tools/src/syn_ext.rs

@@ -0,0 +1,336 @@
+// Copyright 2017-2018 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+// tag::description[]
+//! Extension to syn types, mainly for parsing
+// end::description[]
+
+use syn::parse::{
+	Parse,
+	ParseStream,
+	Result,
+};
+use syn::token::CustomKeyword;
+use proc_macro2::TokenStream as T2;
+use quote::ToTokens;
+use std::iter::once;
+use syn::Ident;
+
+/// stop parsing here getting remaining token as content
+/// Warn duplicate stream (part of)
+#[derive(ParseStruct, ToTokensStruct, Debug)]
+pub struct StopParse {
+	pub inner: T2,
+}
+
+// inner macro really dependant on syn naming convention, do not export
+macro_rules! groups_impl {
+	($name:ident, $tok:ident, $deli:ident, $parse:ident) => {
+
+		#[derive(Debug)]
+		pub struct $name<P> {
+			pub token: syn::token::$tok,
+			pub content: P,
+		}
+
+		impl<P: Parse> Parse for $name<P> {
+			fn parse(input: ParseStream) -> Result<Self> {
+				let syn::group::$name { token, content } = syn::group::$parse(input)?;
+				let content = content.parse()?;
+				Ok($name { token, content, })
+			}
+		}
+
+		impl<P: ToTokens> ToTokens for $name<P> {
+			fn to_tokens(&self, tokens: &mut T2) {
+				let mut inner_stream = T2::new();
+				self.content.to_tokens(&mut inner_stream);
+				let token_tree: proc_macro2::TokenTree =
+					proc_macro2::Group::new(proc_macro2::Delimiter::$deli, inner_stream).into();
+				tokens.extend(once(token_tree));
+			}
+		}
+
+	}
+}
+
+groups_impl!(Braces, Brace, Brace, parse_braces);
+groups_impl!(Brackets, Bracket, Bracket, parse_brackets);
+groups_impl!(Parens, Paren, Parenthesis, parse_parens);
+
+#[derive(Debug)]
+pub struct CustomToken<T>(std::marker::PhantomData<T>);
+
+impl<T: CustomKeyword> Parse for CustomToken<T> {
+	fn parse(input: ParseStream) -> Result<Self> {
+		let ident: syn::Ident = input.parse()?;
+
+		if ident.to_string().as_str() != T::ident() {
+			return Err(syn::parse::Error::new_spanned(ident, "expected another custom token"))
+		}
+		Ok(CustomToken(std::marker::PhantomData))
+	}
+}
+
+impl<T: CustomKeyword> ToTokens for CustomToken<T> {
+	fn to_tokens(&self, tokens: &mut T2) {
+		use std::str::FromStr;
+		tokens.extend(T2::from_str(T::ident()).expect("custom keyword should parse to ident"));
+	}
+}
+
+impl<T: CustomKeyword> CustomKeyword for CustomToken<T> {
+	fn ident() -> &'static str { <T as CustomKeyword>::ident() }
+	fn display() -> &'static str { <T as CustomKeyword>::display() }
+}
+
+#[derive(Debug)]
+pub struct PunctuatedInner<P,T,V> {
+	pub inner: syn::punctuated::Punctuated<P,T>,
+	pub variant: V,
+}
+
+#[derive(Debug)]
+pub struct NoTrailing;
+
+
+#[derive(Debug)]
+pub struct Trailing;
+
+pub type Punctuated<P,T> = PunctuatedInner<P,T,NoTrailing>;
+
+pub type PunctuatedTrailing<P,T> = PunctuatedInner<P,T,Trailing>;
+
+impl<P: Parse, T: Parse + syn::token::Token> Parse for PunctuatedInner<P,T,Trailing> {
+	fn parse(input: ParseStream) -> Result<Self> {
+		Ok(PunctuatedInner {
+			inner: syn::punctuated::Punctuated::parse_separated_nonempty(input)?,
+			variant: Trailing,
+		})
+	}
+}
+
+impl<P: Parse, T: Parse> Parse for PunctuatedInner<P,T,NoTrailing> {
+	fn parse(input: ParseStream) -> Result<Self> {
+		Ok(PunctuatedInner {
+			inner: syn::punctuated::Punctuated::parse_terminated(input)?,
+			variant: NoTrailing,
+		})
+	}
+}
+
+impl<P: ToTokens, T: ToTokens, V> ToTokens for PunctuatedInner<P,T,V> {
+	fn to_tokens(&self, tokens: &mut T2) {
+		self.inner.to_tokens(tokens)
+	}
+}
+
+/// Note that syn Meta is almost fine for use case (lacks only `ToToken`)
+#[derive(Debug, Clone)]
+pub struct Meta {
+	pub inner: syn::Meta,
+}
+
+impl Parse for Meta {
+	fn parse(input: ParseStream) -> Result<Self> {
+		Ok(Meta {
+			inner: syn::Meta::parse(input)?,
+		})
+	}
+}
+
+impl ToTokens for Meta {
+	fn to_tokens(&self, tokens: &mut T2) {
+		match self.inner {
+			syn::Meta::Word(ref ident) => {
+				let ident = ident.clone();
+				let toks = quote!{
+					#[#ident]
+				};
+				tokens.extend(toks);
+			},
+			syn::Meta::List(ref l) => l.to_tokens(tokens),
+			syn::Meta::NameValue(ref n) => n.to_tokens(tokens),
+		}
+	}
+}
+
+#[derive(Debug)]
+pub struct OuterAttributes {
+	pub inner: Vec<syn::Attribute>,
+}
+
+impl Parse for OuterAttributes {
+	fn parse(input: ParseStream) -> Result<Self> {
+		let inner = syn::Attribute::parse_outer(input)?;
+		Ok(OuterAttributes {
+			inner,
+		})
+	}
+}
+
+impl ToTokens for OuterAttributes {
+	fn to_tokens(&self, tokens: &mut T2) {
+		for att in self.inner.iter() {
+			att.to_tokens(tokens);
+		}
+	}
+}
+
+#[derive(Debug)]
+pub struct Seq<P> {
+	pub inner: Vec<P>,
+}
+
+impl<P: Parse> Parse for Seq<P> {
+	// Note that it cost a double parsing (same as enum derive)
+	fn parse(input: ParseStream) -> Result<Self> {
+		let mut inner = Vec::new();
+		loop {
+			let fork = input.fork();
+			let res: Result<P> = fork.parse();
+			match res {
+				Ok(_item) => {
+					// move cursor
+					let item: P = input.parse().expect("Same parsing ran before");
+					inner.push(item);
+				},
+				Err(_e) => break,
+			}
+		}
+		Ok(Seq { inner })
+	}
+}
+
+impl<P: ToTokens> ToTokens for Seq<P> {
+	fn to_tokens(&self, tokens: &mut T2) {
+		for p in self.inner.iter() {
+			p.to_tokens(tokens);
+		}
+	}
+}
+
+pub fn extract_type_option(typ: &syn::Type) -> Option<T2> {
+	if let syn::Type::Path(ref path) = typ {
+		path.path.segments.last().and_then(|v| {
+			if v.value().ident == "Option" {
+				if let syn::PathArguments::AngleBracketed(ref a) = v.value().arguments {
+					let args = &a.args;
+					Some(quote!{ #args })
+				} else {
+					None
+				}
+			} else {
+				None
+			}
+		})
+	} else {
+		None
+	}
+}
+
+pub fn is_parametric_type_def(typ: &syn::Type, default: bool) -> bool {
+	match *typ {
+		syn::Type::Path(ref path) => {
+			path.path.segments.iter().any(|v| {
+				if let syn::PathArguments::AngleBracketed(..) = v.arguments {
+					true
+				} else {
+					false
+				}
+			})
+		},
+		syn::Type::Slice(ref inner) => is_parametric_type_def(&inner.elem, default),
+		syn::Type::Array(ref inner) => is_parametric_type_def(&inner.elem, default),
+		syn::Type::Ptr(ref inner) => is_parametric_type_def(&inner.elem, default),
+		syn::Type::Reference(ref inner) => is_parametric_type_def(&inner.elem, default),
+		syn::Type::BareFn(ref inner) => inner.variadic.is_some(),
+		syn::Type::Never(..) => false,
+		syn::Type::Tuple(ref inner) =>
+			inner.elems.iter().any(|t| is_parametric_type_def(t, default)),
+		syn::Type::TraitObject(..) => true,
+		syn::Type::ImplTrait(..) => true,
+		syn::Type::Paren(ref inner) => is_parametric_type_def(&inner.elem, default),
+		syn::Type::Group(ref inner) => is_parametric_type_def(&inner.elem, default),
+		syn::Type::Infer(..) => true,
+		syn::Type::Macro(..) => default,
+		syn::Type::Verbatim(..) => default,
+	}
+}
+
+/// check if type has any type parameter, defaults to true for some cases.
+pub fn is_parametric_type(typ: &syn::Type) -> bool {
+	is_parametric_type_def(typ, true)
+}
+
+fn has_parametric_type_def_in_path(path: &syn::Path, ident: &Ident, default: bool) -> bool {
+	path.segments.iter().any(|v| {
+		if ident == &v.ident {
+			return true;
+		}
+		if let syn::PathArguments::AngleBracketed(ref a) = v.arguments {
+			for arg in a.args.iter() {
+				if let syn::GenericArgument::Type(ref typ) = arg {
+					if has_parametric_type_def(typ, ident, default) {
+						return true;
+					}
+				}
+				// potentially missing matches here
+			}
+			false
+		} else {
+			false
+		}
+	})
+
+}
+pub fn has_parametric_type_def(typ: &syn::Type, ident: &Ident, default: bool) -> bool {
+	match *typ {
+		syn::Type::Path(ref path) => has_parametric_type_def_in_path(&path.path, ident, default),
+		syn::Type::Slice(ref inner) => has_parametric_type_def(&inner.elem, ident, default),
+		syn::Type::Array(ref inner) => has_parametric_type_def(&inner.elem, ident, default),
+		syn::Type::Ptr(ref inner) => has_parametric_type_def(&inner.elem, ident, default),
+		syn::Type::Reference(ref inner) => has_parametric_type_def(&inner.elem, ident, default),
+		syn::Type::BareFn(ref inner) => inner.variadic.is_some(),
+		syn::Type::Never(..) => false,
+		syn::Type::Tuple(ref inner) =>
+			inner.elems.iter().any(|t| has_parametric_type_def(t, ident, default)),
+		syn::Type::TraitObject(ref to) => {
+			to.bounds.iter().any(|bound| {
+				if let syn::TypeParamBound::Trait(ref t) = bound {
+					has_parametric_type_def_in_path(&t.path, ident, default)
+				} else { false }
+			})
+		},
+		syn::Type::ImplTrait(ref it) => {
+			it.bounds.iter().any(|bound| {
+				if let syn::TypeParamBound::Trait(ref t) = bound {
+					has_parametric_type_def_in_path(&t.path, ident, default)
+				} else { false }
+			})
+		},
+		syn::Type::Paren(ref inner) => has_parametric_type_def(&inner.elem, ident, default),
+		syn::Type::Group(ref inner) => has_parametric_type_def(&inner.elem, ident, default),
+		syn::Type::Infer(..) => default,
+		syn::Type::Macro(..) => default,
+		syn::Type::Verbatim(..) => default,
+	}
+}
+
+/// check if type has a type parameter, defaults to true for some cases.
+pub fn has_parametric_type(typ: &syn::Type, ident: &Ident) -> bool {
+	has_parametric_type_def(typ, ident, true)
+}