Versioning for #[runtime-interface] (#5328)

* versionned runtime-interface

* use only one additional wasm blob

* alter docs

* formatting, naming and docs

* add comment for test

* version duplicate err

* RuntimeInterfaceItem -> RuntimeInterfaceFunction<

* test naming

* version checking

* remove spaces

* Update primitives/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs

Co-Authored-By: Bastian Köcher <bkchr@users.noreply.github.com>

* remove sanity checks and reduce coverage

* add doc comment

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

substrate/primitives/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs

@@ -30,7 +30,7 @@
 
 use crate::utils::{
 	generate_crate_access, create_exchangeable_host_function_ident, get_function_arguments,
-	get_function_argument_names, get_trait_methods,
+	get_function_argument_names, get_runtime_interface, create_function_ident_with_version,
 };
 
 use syn::{
@@ -47,19 +47,40 @@ use std::iter;
 /// of the trait method.
 pub fn generate(trait_def: &ItemTrait, is_wasm_only: bool) -> Result<TokenStream> {
 	let trait_name = &trait_def.ident;
-	get_trait_methods(trait_def).try_fold(TokenStream::new(), |mut t, m| {
-		t.extend(function_for_method(trait_name, m, is_wasm_only)?);
+	let runtime_interface = get_runtime_interface(trait_def)?;
+
+	// latest version dispatch
+	let token_stream: Result<TokenStream> = runtime_interface.latest_versions()
+		.try_fold(
+			TokenStream::new(),
+			|mut t, (latest_version, method)| {
+				t.extend(function_for_method(method, latest_version, is_wasm_only)?);
+				Ok(t)
+			}
+		);
+
+	// earlier versions compatibility dispatch (only std variant)
+	let result: Result<TokenStream> = runtime_interface.all_versions().try_fold(token_stream?, |mut t, (version, method)|
+	{
+		t.extend(function_std_impl(trait_name, method, version, is_wasm_only)?);
 		Ok(t)
-	})
+	});
+
+	result
 }
 
 /// Generates the bare function implementation for the given method for the host and wasm side.
 fn function_for_method(
-	trait_name: &Ident,
 	method: &TraitItemMethod,
+	latest_version: u32,
 	is_wasm_only: bool,
 ) -> Result<TokenStream> {
-	let std_impl = function_std_impl(trait_name, method, is_wasm_only)?;
+	let std_impl = if !is_wasm_only {
+		function_std_latest_impl(method, latest_version)?
+	} else {
+		quote!()
+	};
+
 	let no_std_impl = function_no_std_impl(method)?;
 
 	Ok(
@@ -78,7 +99,7 @@ fn function_no_std_impl(method: &TraitItemMethod) -> Result<TokenStream> {
 	let args = get_function_arguments(&method.sig);
 	let arg_names = get_function_argument_names(&method.sig);
 	let return_value = &method.sig.output;
-	let attrs = &method.attrs;
+	let attrs = method.attrs.iter().filter(|a| !a.path.is_ident("version"));
 
 	Ok(
 		quote! {
@@ -92,13 +113,40 @@ fn function_no_std_impl(method: &TraitItemMethod) -> Result<TokenStream> {
 	)
 }
 
+/// Generate call to latest function version for `cfg((feature = "std")`
+///
+/// This should generate simple `fn func(..) { func_version_<latest_version>(..) }`.
+fn function_std_latest_impl(
+	method: &TraitItemMethod,
+	latest_version: u32,
+) -> Result<TokenStream> {
+	let function_name = &method.sig.ident;
+	let args = get_function_arguments(&method.sig).map(FnArg::Typed);
+	let arg_names = get_function_argument_names(&method.sig).collect::<Vec<_>>();
+	let return_value = &method.sig.output;
+	let attrs = method.attrs.iter().filter(|a| !a.path.is_ident("version"));
+	let latest_function_name = create_function_ident_with_version(&method.sig.ident, latest_version);
+
+	Ok(quote_spanned! { method.span() =>
+		#[cfg(feature = "std")]
+		#( #attrs )*
+		pub fn #function_name( #( #args, )* ) #return_value {
+			#latest_function_name(
+				#( #arg_names, )*
+			)
+		}
+	})
+}
+
 /// Generates the bare function implementation for `cfg(feature = "std")`.
 fn function_std_impl(
 	trait_name: &Ident,
 	method: &TraitItemMethod,
+	version: u32,
 	is_wasm_only: bool,
 ) -> Result<TokenStream> {
-	let function_name = &method.sig.ident;
+	let function_name = create_function_ident_with_version(&method.sig.ident, version);
+
 	let crate_ = generate_crate_access();
 	let args = get_function_arguments(&method.sig).map(FnArg::Typed).chain(
 		// Add the function context as last parameter when this is a wasm only interface.
@@ -115,16 +163,15 @@ fn function_std_impl(
 		).take(1),
 	);
 	let return_value = &method.sig.output;
-	let attrs = &method.attrs;
+	let attrs = method.attrs.iter().filter(|a| !a.path.is_ident("version"));
 	// Don't make the function public accessible when this is a wasm only interface.
-	let vis = if is_wasm_only { quote!() } else { quote!(pub) };
-	let call_to_trait = generate_call_to_trait(trait_name, method, is_wasm_only);
+	let call_to_trait = generate_call_to_trait(trait_name, method, version, is_wasm_only);
 
 	Ok(
 		quote_spanned! { method.span() =>
 			#[cfg(feature = "std")]
 			#( #attrs )*
-			#vis fn #function_name( #( #args, )* ) #return_value {
+			fn #function_name( #( #args, )* ) #return_value {
 				#call_to_trait
 			}
 		}
@@ -135,10 +182,11 @@ fn function_std_impl(
 fn generate_call_to_trait(
 	trait_name: &Ident,
 	method: &TraitItemMethod,
+	version: u32,
 	is_wasm_only: bool,
 ) -> TokenStream {
 	let crate_ = generate_crate_access();
-	let method_name = &method.sig.ident;
+	let method_name = create_function_ident_with_version(&method.sig.ident, version);
 	let expect_msg = format!(
 		"`{}` called outside of an Externalities-provided environment.",
 		method_name,

substrate/primitives/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs

@@ -23,13 +23,13 @@
 use crate::utils::{
 	generate_crate_access, create_host_function_ident, get_function_argument_names,
 	get_function_argument_types_without_ref, get_function_argument_types_ref_and_mut,
-	get_function_argument_names_and_types_without_ref, get_trait_methods, get_function_arguments,
-	get_function_argument_types, create_exchangeable_host_function_ident,
+	get_function_argument_names_and_types_without_ref, get_function_arguments,
+	get_function_argument_types, create_exchangeable_host_function_ident, get_runtime_interface,
+	create_function_ident_with_version,
 };
 
 use syn::{
-	ItemTrait, TraitItemMethod, Result, ReturnType, Ident, TraitItem, Pat, Error, Signature,
-	spanned::Spanned,
+	ItemTrait, TraitItemMethod, Result, ReturnType, Ident, Pat, Error, Signature, spanned::Spanned,
 };
 
 use proc_macro2::{TokenStream, Span};
@@ -44,13 +44,15 @@ use std::iter::{Iterator, self};
 /// implementations for the host functions on the host.
 pub fn generate(trait_def: &ItemTrait, is_wasm_only: bool) -> Result<TokenStream> {
 	let trait_name = &trait_def.ident;
-	let extern_host_function_impls = get_trait_methods(trait_def)
-		.try_fold(TokenStream::new(), |mut t, m| {
-			t.extend(generate_extern_host_function(m, trait_name)?);
+	let extern_host_function_impls = get_runtime_interface(trait_def)?
+		.latest_versions()
+		.try_fold(TokenStream::new(), |mut t, (version, method)| {
+			t.extend(generate_extern_host_function(method, version, trait_name)?);
 			Ok::<_, Error>(t)
 		})?;
-	let exchangeable_host_functions = get_trait_methods(trait_def)
-		.try_fold(TokenStream::new(), |mut t, m| {
+	let exchangeable_host_functions = get_runtime_interface(trait_def)?
+		.latest_versions()
+		.try_fold(TokenStream::new(), |mut t, (_, m)| {
 			t.extend(generate_exchangeable_host_function(m)?);
 			Ok::<_, Error>(t)
 		})?;
@@ -76,7 +78,7 @@ pub fn generate(trait_def: &ItemTrait, is_wasm_only: bool) -> Result<TokenStream
 }
 
 /// Generate the extern host function for the given method.
-fn generate_extern_host_function(method: &TraitItemMethod, trait_name: &Ident) -> Result<TokenStream> {
+fn generate_extern_host_function(method: &TraitItemMethod, version: u32, trait_name: &Ident) -> Result<TokenStream> {
 	let crate_ = generate_crate_access();
 	let args = get_function_arguments(&method.sig);
 	let arg_types = get_function_argument_types_without_ref(&method.sig);
@@ -85,7 +87,7 @@ fn generate_extern_host_function(method: &TraitItemMethod, trait_name: &Ident) -
 	let arg_names2 = get_function_argument_names(&method.sig);
 	let arg_names3 = get_function_argument_names(&method.sig);
 	let function = &method.sig.ident;
-	let ext_function = create_host_function_ident(&method.sig.ident, trait_name);
+	let ext_function = create_host_function_ident(&method.sig.ident, version, trait_name);
 	let doc_string = format!(
 		" Default extern host function implementation for [`super::{}`].",
 		method.sig.ident,
@@ -157,14 +159,12 @@ fn generate_exchangeable_host_function(method: &TraitItemMethod) -> Result<Token
 /// implementations for the extern host functions.
 fn generate_host_functions_struct(trait_def: &ItemTrait, is_wasm_only: bool) -> Result<TokenStream> {
 	let crate_ = generate_crate_access();
-	let host_functions = trait_def
-		.items
-		.iter()
-		.filter_map(|i| match i {
-			TraitItem::Method(ref method) => Some(method),
-			_ => None,
-		})
-		.map(|m| generate_host_function_implementation(&trait_def.ident, m, is_wasm_only))
+
+	let host_functions = get_runtime_interface(trait_def)?
+		.all_versions()
+		.map(|(version, method)|
+			generate_host_function_implementation(&trait_def.ident, method, version, is_wasm_only)
+		)
 		.collect::<Result<Vec<_>>>()?;
 
 	Ok(
@@ -191,9 +191,10 @@ fn generate_host_functions_struct(trait_def: &ItemTrait, is_wasm_only: bool) ->
 fn generate_host_function_implementation(
 	trait_name: &Ident,
 	method: &TraitItemMethod,
+	version: u32,
 	is_wasm_only: bool,
 ) -> Result<TokenStream> {
-	let name = create_host_function_ident(&method.sig.ident, trait_name).to_string();
+	let name = create_host_function_ident(&method.sig.ident, version, trait_name).to_string();
 	let struct_name = Ident::new(&name.to_pascal_case(), Span::call_site());
 	let crate_ = generate_crate_access();
 	let signature = generate_wasm_interface_signature_for_host_function(&method.sig)?;
@@ -202,7 +203,7 @@ fn generate_host_function_implementation(
 		trait_name,
 	).collect::<Result<Vec<_>>>()?;
 	let ffi_to_host_values = generate_ffi_to_host_value(&method.sig).collect::<Result<Vec<_>>>()?;
-	let host_function_call = generate_host_function_call(&method.sig, is_wasm_only);
+	let host_function_call = generate_host_function_call(&method.sig, version, is_wasm_only);
 	let into_preallocated_ffi_value = generate_into_preallocated_ffi_value(&method.sig)?;
 	let convert_return_value = generate_return_value_into_wasm_value(&method.sig);
 
@@ -211,7 +212,6 @@ fn generate_host_function_implementation(
 			{
 				struct #struct_name;
 
-				#[allow(unused)]
 				impl #crate_::sp_wasm_interface::Function for #struct_name {
 					fn name(&self) -> &str {
 						#name
@@ -322,8 +322,8 @@ fn generate_ffi_to_host_value<'a>(
 }
 
 /// Generate the code to call the host function and the ident that stores the result.
-fn generate_host_function_call(sig: &Signature, is_wasm_only: bool) -> TokenStream {
-	let host_function_name = &sig.ident;
+fn generate_host_function_call(sig: &Signature, version: u32, is_wasm_only: bool) -> TokenStream {
+	let host_function_name = create_function_ident_with_version(&sig.ident, version);
 	let result_var_name = generate_host_function_result_var_name(&sig.ident);
 	let ref_and_mut = get_function_argument_types_ref_and_mut(sig).map(|ram|
 		ram.map(|(vr, vm)| quote!(#vr #vm))

substrate/primitives/runtime-interface/proc-macro/src/runtime_interface/trait_decl_impl.rs

@@ -17,10 +17,15 @@
 //! Checks the trait declaration, makes the trait declaration module local, removes all method
 //! default implementations and implements the trait for `&mut dyn Externalities`.
 
-use crate::utils::{generate_crate_access, get_function_argument_types_without_ref};
+use crate::utils::{
+	generate_crate_access,
+	get_function_argument_types_without_ref,
+	get_runtime_interface,
+	create_function_ident_with_version,
+};
 
 use syn::{
-	ItemTrait, TraitItemMethod, Result, TraitItem, Error, fold::{self, Fold}, spanned::Spanned,
+	ItemTrait, TraitItemMethod, Result, Error, fold::{self, Fold}, spanned::Spanned,
 	Visibility, Receiver, Type, Generics,
 };
 
@@ -32,7 +37,7 @@ use quote::quote;
 /// essential definition and implement this essential definition for `dyn Externalities`.
 pub fn process(trait_def: &ItemTrait, is_wasm_only: bool) -> Result<TokenStream> {
 	let impl_trait = impl_trait_for_externalities(trait_def, is_wasm_only)?;
-	let essential_trait_def = ToEssentialTraitDef::convert(trait_def.clone())?;
+	let essential_trait_def = declare_essential_trait(trait_def)?;
 
 	Ok(
 		quote! {
@@ -48,29 +53,35 @@ pub fn process(trait_def: &ItemTrait, is_wasm_only: bool) -> Result<TokenStream>
 struct ToEssentialTraitDef {
 	/// All errors found while doing the conversion.
 	errors: Vec<Error>,
+	methods: Vec<TraitItemMethod>,
 }
 
 impl ToEssentialTraitDef {
-	/// Convert the given trait definition to the essential trait definition.
-	fn convert(trait_def: ItemTrait) -> Result<ItemTrait> {
-		let mut folder = ToEssentialTraitDef {
-			errors: Vec::new(),
-		};
+	fn new() -> Self {
+		ToEssentialTraitDef { errors: vec![], methods: vec![] }
+	}
 
-		let res = folder.fold_item_trait(trait_def);
-
-		if let Some(first_error) = folder.errors.pop() {
+	fn into_methods(self) -> Result<Vec<TraitItemMethod>> {
+		let mut errors = self.errors;
+		let methods = self.methods;
+		if let Some(first_error) = errors.pop() {
 			Err(
-				folder.errors.into_iter().fold(first_error, |mut o, n| {
+				errors.into_iter().fold(first_error, |mut o, n| {
 					o.combine(n);
 					o
 				})
 			)
 		} else {
-			Ok(res)
+			Ok(methods)
 		}
 	}
 
+	fn process(&mut self, method: &TraitItemMethod, version: u32) {
+		let mut folded = self.fold_trait_item_method(method.clone());
+		folded.sig.ident = create_function_ident_with_version(&folded.sig.ident, version);
+		self.methods.push(folded);
+	}
+
 	fn push_error<S: Spanned>(&mut self, span: &S, msg: &str) {
 		self.errors.push(Error::new(span.span(), msg));
 	}
@@ -98,6 +109,8 @@ impl Fold for ToEssentialTraitDef {
 
 		self.error_on_generic_parameters(&method.sig.generics);
 
+		method.attrs.retain(|a| !a.path.is_ident("version"));
+
 		fold::fold_trait_item_method(self, method)
 	}
 
@@ -117,17 +130,40 @@ impl Fold for ToEssentialTraitDef {
 	}
 }
 
+fn declare_essential_trait(trait_def: &ItemTrait) -> Result<TokenStream> {
+	let trait_ = &trait_def.ident;
+
+	if let Some(param) = trait_def.generics.params.first() {
+		return Err(Error::new(param.span(), "Generic parameters not supported."))
+	}
+
+	let interface = get_runtime_interface(trait_def)?;
+	let mut folder = ToEssentialTraitDef::new();
+	for (version, interface_method) in interface.all_versions() {
+		folder.process(interface_method, version);
+	}
+	let methods = folder.into_methods()?;
+
+	Ok(
+		quote! {
+			trait #trait_ {
+				#( #methods )*
+			}
+		}
+	)
+}
+
 /// Implements the given trait definition for `dyn Externalities`.
 fn impl_trait_for_externalities(trait_def: &ItemTrait, is_wasm_only: bool) -> Result<TokenStream> {
 	let trait_ = &trait_def.ident;
 	let crate_ = generate_crate_access();
-	let methods = trait_def
-		.items
-		.iter()
-		.filter_map(|i| match i {
-			TraitItem::Method(ref method) => Some(method),
-			_ => None,
-		});
+	let interface = get_runtime_interface(trait_def)?;
+	let methods = interface.all_versions().map(|(version, method)| {
+		let mut cloned = method.clone();
+		cloned.attrs.retain(|a| !a.path.is_ident("version"));
+		cloned.sig.ident = create_function_ident_with_version(&cloned.sig.ident, version);
+		cloned
+	});
 
 	let impl_type = if is_wasm_only {
 		quote!( &mut dyn #crate_::sp_wasm_interface::FunctionContext )

substrate/primitives/runtime-interface/proc-macro/src/utils.rs

@@ -20,17 +20,60 @@ use proc_macro2::{TokenStream, Span};
 
 use syn::{
 	Ident, Error, Signature, Pat, PatType, FnArg, Type, token, TraitItemMethod, ItemTrait,
-	TraitItem, parse_quote, spanned::Spanned,
+	TraitItem, parse_quote, spanned::Spanned, Result, Meta, NestedMeta, Lit, Attribute,
 };
 
 use proc_macro_crate::crate_name;
 
 use std::env;
+use std::collections::{BTreeMap, btree_map::Entry};
 
 use quote::quote;
 
 use inflector::Inflector;
 
+/// Runtime interface function with all associated versions of this function.
+pub struct RuntimeInterfaceFunction<'a> {
+	latest_version: u32,
+	versions: BTreeMap<u32, &'a TraitItemMethod>,
+}
+
+impl<'a> RuntimeInterfaceFunction<'a> {
+	fn new(version: u32, trait_item: &'a TraitItemMethod) -> Self {
+		Self {
+			latest_version: version,
+			versions: {
+				let mut res = BTreeMap::new();
+				res.insert(version, trait_item);
+				res
+			},
+		}
+	}
+
+	pub fn latest_version(&self) -> (u32, &TraitItemMethod) {
+		(
+			self.latest_version,
+			self.versions.get(&self.latest_version)
+				.expect("If latest_version has a value, the key with this value is in the versions; qed")
+		)
+	}
+}
+
+/// All functions of a runtime interface grouped by the function names.
+pub struct RuntimeInterface<'a> {
+	items: BTreeMap<syn::Ident, RuntimeInterfaceFunction<'a>>,
+}
+
+impl<'a> RuntimeInterface<'a> {
+	pub fn latest_versions(&self) -> impl Iterator<Item = (u32, &TraitItemMethod)> {
+		self.items.iter().map(|(_, item)| item.latest_version())
+	}
+
+	pub fn all_versions(&self) -> impl Iterator<Item = (u32, &TraitItemMethod)> {
+		self.items.iter().flat_map(|(_, item)| item.versions.iter()).map(|(v, i)| (*v, *i))
+	}
+ }
+
 /// Generates the include for the runtime-interface crate.
 pub fn generate_runtime_interface_include() -> TokenStream {
 	if env::var("CARGO_PKG_NAME").unwrap() == "sp-runtime-interface" {
@@ -67,12 +110,25 @@ pub fn create_exchangeable_host_function_ident(name: &Ident) -> Ident {
 }
 
 /// Create the host function identifier for the given function name.
-pub fn create_host_function_ident(name: &Ident, trait_name: &Ident) -> Ident {
+pub fn create_host_function_ident(name: &Ident, version: u32, trait_name: &Ident) -> Ident {
 	Ident::new(
 		&format!(
-			"ext_{}_{}_version_1",
+			"ext_{}_{}_version_{}",
 			trait_name.to_string().to_snake_case(),
 			name,
+			version,
+		),
+		Span::call_site(),
+	)
+}
+
+/// Create the host function identifier for the given function name.
+pub fn create_function_ident_with_version(name: &Ident, version: u32) -> Ident {
+	Ident::new(
+		&format!(
+			"{}_version_{}",
+			name,
+			version,
 		),
 		Span::call_site(),
 	)
@@ -151,7 +207,7 @@ pub fn get_function_argument_types_ref_and_mut<'a>(
 }
 
 /// Returns an iterator over all trait methods for the given trait definition.
-pub fn get_trait_methods<'a>(trait_def: &'a ItemTrait) -> impl Iterator<Item = &'a TraitItemMethod> {
+fn get_trait_methods<'a>(trait_def: &'a ItemTrait) -> impl Iterator<Item = &'a TraitItemMethod> {
 	trait_def
 		.items
 		.iter()
@@ -160,3 +216,85 @@ pub fn get_trait_methods<'a>(trait_def: &'a ItemTrait) -> impl Iterator<Item = &
 			_ => None,
 		})
 }
+
+/// Parse version attribute.
+///
+/// Returns error if it is in incorrent format. Correct format is only `#[version(X)]`.
+fn parse_version_attribute(version: &Attribute) -> Result<u32> {
+	let meta = version.parse_meta()?;
+
+	let err = Err(Error::new(
+			meta.span(),
+			"Unexpected `version` attribute. The supported format is `#[version(1)]`",
+		)
+	);
+
+	match meta {
+		Meta::List(list) => {
+			if list.nested.len() != 1 {
+				err
+			} else if let Some(NestedMeta::Lit(Lit::Int(i))) = list.nested.first() {
+				i.base10_parse()
+			} else {
+				err
+			}
+		},
+		_ => err,
+	}
+}
+
+/// Return item version (`#[version(X)]`) attribute, if present.
+fn get_item_version(item: &TraitItemMethod) -> Result<Option<u32>> {
+	item.attrs.iter().find(|attr| attr.path.is_ident("version"))
+		.map(|attr| parse_version_attribute(attr))
+		.transpose()
+}
+
+/// Returns all runtime interface members, with versions.
+pub fn get_runtime_interface<'a>(trait_def: &'a ItemTrait)
+	-> Result<RuntimeInterface<'a>>
+{
+	let mut functions: BTreeMap<syn::Ident, RuntimeInterfaceFunction<'a>> = BTreeMap::new();
+
+	for item in get_trait_methods(trait_def) {
+		let name = item.sig.ident.clone();
+		let version = get_item_version(item)?.unwrap_or(1);
+
+		match functions.entry(name.clone()) {
+			Entry::Vacant(entry) => { entry.insert(RuntimeInterfaceFunction::new(version, item)); },
+			Entry::Occupied(mut entry) => {
+				if let Some(existing_item) = entry.get().versions.get(&version) {
+					let mut err = Error::new(
+						item.span(),
+						"Duplicated version attribute",
+					);
+					err.combine(Error::new(
+						existing_item.span(),
+						"Previous version with the same number defined here",
+					));
+
+					return Err(err);
+				}
+
+				let interface_item = entry.get_mut();
+				if interface_item.latest_version < version { interface_item.latest_version = version; }
+				interface_item.versions.insert(version, item);
+			}
+		}
+	}
+
+	for function in functions.values() {
+		let mut next_expected = 1;
+		for (version, item) in function.versions.iter() {
+			if next_expected != *version {
+				return Err(Error::new(
+					item.span(),
+					format!("Unexpected version attribute: missing version '{}' for this function", next_expected),
+				));
+			}
+			next_expected += 1;
+		}
+	}
+
+	Ok(RuntimeInterface { items: functions })
+}