sp-api: Support expanding the macro code (#13573)

* sp-api: Support expanding the macro code

This pr introduces the `expander` crate to expand the generated source code into a file. This gives
better error reporting when trying to fix issues in the macro itself as Rustc will point to the line
in this file. The feature can be enabled by setting `SP_API_EXPAND=1` at compile time.

Besides that the generated code is changed to fix warnings in the exanped version.

* Fixes

substrate/primitives/api/proc-macro/src/common.rs

@@ -18,9 +18,6 @@
 /// The ident used for the block generic parameter.
 pub const BLOCK_GENERIC_IDENT: &str = "Block";
 
-/// Unique identifier used to make the hidden includes unique for this macro.
-pub const HIDDEN_INCLUDES_ID: &str = "DECL_RUNTIME_APIS";
-
 /// The `core_trait` attribute.
 pub const CORE_TRAIT_ATTRIBUTE: &str = "core_trait";
 /// The `api_version` attribute.

substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs

@@ -17,14 +17,14 @@
 
 use crate::utils::{
 	extract_parameter_names_types_and_borrows, fold_fn_decl_for_client_side, generate_crate_access,
-	generate_hidden_includes, generate_runtime_mod_name_for_trait, parse_runtime_api_version,
-	prefix_function_with_trait, replace_wild_card_parameter_names, return_type_extract_type,
-	versioned_trait_name, AllowSelfRefInParameters,
+	generate_runtime_mod_name_for_trait, parse_runtime_api_version, prefix_function_with_trait,
+	replace_wild_card_parameter_names, return_type_extract_type, versioned_trait_name,
+	AllowSelfRefInParameters,
 };
 
 use crate::common::{
 	API_VERSION_ATTRIBUTE, BLOCK_GENERIC_IDENT, CHANGED_IN_ATTRIBUTE, CORE_TRAIT_ATTRIBUTE,
-	HIDDEN_INCLUDES_ID, RENAMED_ATTRIBUTE, SUPPORTED_ATTRIBUTE_NAMES,
+	RENAMED_ATTRIBUTE, SUPPORTED_ATTRIBUTE_NAMES,
 };
 
 use proc_macro2::{Span, TokenStream};
@@ -62,7 +62,7 @@ impl Parse for RuntimeApiDecls {
 
 /// Extend the given generics with `Block: BlockT` as first generic parameter.
 fn extend_generics_with_block(generics: &mut Generics) {
-	let c = generate_crate_access(HIDDEN_INCLUDES_ID);
+	let c = generate_crate_access();
 
 	generics.lt_token = Some(Default::default());
 	generics.params.insert(0, parse_quote!( Block: #c::BlockT ));
@@ -298,7 +298,7 @@ fn generate_runtime_decls(decls: &[ItemTrait]) -> Result<TokenStream> {
 			#[allow(dead_code)]
 			#[allow(deprecated)]
 			pub mod #mod_name {
-				use super::*;
+				pub use super::*;
 
 				#( #versioned_api_traits )*
 
@@ -495,10 +495,10 @@ impl<'a> ToClientSideDecl<'a> {
 					__runtime_api_at_param__,
 					#context,
 					__runtime_api_impl_params_encoded__,
-					&|version| {
+					&|_version| {
 						#(
 							// Check if we need to call the function by an old name.
-							if version.apis.iter().any(|(s, v)| {
+							if _version.apis.iter().any(|(s, v)| {
 								s == &#runtime_mod::ID && *v < #versions
 							}) {
 								return #old_names
@@ -569,7 +569,7 @@ fn generate_runtime_api_version(version: u32) -> TokenStream {
 /// Generates the implementation of `RuntimeApiInfo` for the given trait.
 fn generate_runtime_info_impl(trait_: &ItemTrait, version: u64) -> TokenStream {
 	let trait_name = &trait_.ident;
-	let crate_ = generate_crate_access(HIDDEN_INCLUDES_ID);
+	let crate_ = generate_crate_access();
 	let id = generate_runtime_api_id(&trait_name.to_string());
 	let version = generate_runtime_api_version(version as u32);
 
@@ -620,7 +620,7 @@ fn generate_client_side_decls(decls: &[ItemTrait]) -> Result<TokenStream> {
 	for decl in decls {
 		let decl = decl.clone();
 
-		let crate_ = generate_crate_access(HIDDEN_INCLUDES_ID);
+		let crate_ = generate_crate_access();
 		let block_hash = quote!( <Block as #crate_::BlockT>::Hash );
 		let mut found_attributes = HashMap::new();
 		let mut errors = Vec::new();
@@ -777,15 +777,20 @@ pub fn decl_runtime_apis_impl(input: proc_macro::TokenStream) -> proc_macro::Tok
 fn decl_runtime_apis_impl_inner(api_decls: &[ItemTrait]) -> Result<TokenStream> {
 	check_trait_decls(api_decls)?;
 
-	let hidden_includes = generate_hidden_includes(HIDDEN_INCLUDES_ID);
 	let runtime_decls = generate_runtime_decls(api_decls)?;
 	let client_side_decls = generate_client_side_decls(api_decls)?;
 
-	Ok(quote!(
-		#hidden_includes
-
+	let decl = quote! {
 		#runtime_decls
 
 		#client_side_decls
-	))
+	};
+
+	let decl = expander::Expander::new("decl_runtime_apis")
+		.dry(std::env::var("SP_API_EXPAND").is_err())
+		.verbose(true)
+		.write_to_out_dir(decl)
+		.expect("Does not fail because of IO in OUT_DIR; qed");
+
+	Ok(decl)
 }

substrate/primitives/api/proc-macro/src/impl_runtime_apis.rs

@@ -17,7 +17,7 @@
 
 use crate::utils::{
 	extract_all_signature_types, extract_block_type_from_trait_path, extract_impl_trait,
-	extract_parameter_names_types_and_borrows, generate_crate_access, generate_hidden_includes,
+	extract_parameter_names_types_and_borrows, generate_crate_access,
 	generate_runtime_mod_name_for_trait, parse_runtime_api_version, prefix_function_with_trait,
 	versioned_trait_name, AllowSelfRefInParameters, RequireQualifiedTraitPath,
 };
@@ -38,9 +38,6 @@ use syn::{
 
 use std::collections::HashSet;
 
-/// Unique identifier used to make the hidden includes unique for this macro.
-const HIDDEN_INCLUDES_ID: &str = "IMPL_RUNTIME_APIS";
-
 /// The structure used for parsing the runtime api implementations.
 struct RuntimeApiImpls {
 	impls: Vec<ItemImpl>,
@@ -73,7 +70,7 @@ fn generate_impl_call(
 	let params =
 		extract_parameter_names_types_and_borrows(signature, AllowSelfRefInParameters::No)?;
 
-	let c = generate_crate_access(HIDDEN_INCLUDES_ID);
+	let c = generate_crate_access();
 	let fn_name = &signature.ident;
 	let fn_name_str = fn_name.to_string();
 	let pnames = params.iter().map(|v| &v.0);
@@ -81,15 +78,33 @@ fn generate_impl_call(
 	let ptypes = params.iter().map(|v| &v.1);
 	let pborrow = params.iter().map(|v| &v.2);
 
+	let decode_params = if params.is_empty() {
+		quote!()
+	} else {
+		let let_binding = if params.len() == 1 {
+			quote! {
+				let #( #pnames )* : #( #ptypes )*
+			}
+		} else {
+			quote! {
+				let ( #( #pnames ),* ) : ( #( #ptypes ),* )
+			}
+		};
+
+		quote!(
+			#let_binding =
+				match #c::DecodeLimit::decode_all_with_depth_limit(
+					#c::MAX_EXTRINSIC_DEPTH,
+					&mut #input,
+				) {
+					Ok(res) => res,
+					Err(e) => panic!("Bad input data provided to {}: {}", #fn_name_str, e),
+				};
+		)
+	};
+
 	Ok(quote!(
-		let (#( #pnames ),*) : ( #( #ptypes ),* ) =
-			match #c::DecodeLimit::decode_all_with_depth_limit(
-				#c::MAX_EXTRINSIC_DEPTH,
-				&mut #input,
-			) {
-				Ok(res) => res,
-				Err(e) => panic!("Bad input data provided to {}: {}", #fn_name_str, e),
-			};
+		#decode_params
 
 		#[allow(deprecated)]
 		<#runtime as #impl_trait>::#fn_name(#( #pborrow #pnames2 ),*)
@@ -134,8 +149,8 @@ fn generate_impl_calls(
 
 /// Generate the dispatch function that is used in native to call into the runtime.
 fn generate_dispatch_function(impls: &[ItemImpl]) -> Result<TokenStream> {
-	let data = Ident::new("__sp_api__input_data", Span::call_site());
-	let c = generate_crate_access(HIDDEN_INCLUDES_ID);
+	let data = Ident::new("_sp_api_input_data_", Span::call_site());
+	let c = generate_crate_access();
 	let impl_calls =
 		generate_impl_calls(impls, &data)?
 			.into_iter()
@@ -161,7 +176,7 @@ fn generate_dispatch_function(impls: &[ItemImpl]) -> Result<TokenStream> {
 /// Generate the interface functions that are used to call into the runtime in wasm.
 fn generate_wasm_interface(impls: &[ItemImpl]) -> Result<TokenStream> {
 	let input = Ident::new("input", Span::call_site());
-	let c = generate_crate_access(HIDDEN_INCLUDES_ID);
+	let c = generate_crate_access();
 
 	let impl_calls =
 		generate_impl_calls(impls, &input)?
@@ -195,7 +210,7 @@ fn generate_wasm_interface(impls: &[ItemImpl]) -> Result<TokenStream> {
 }
 
 fn generate_runtime_api_base_structures() -> Result<TokenStream> {
-	let crate_ = generate_crate_access(HIDDEN_INCLUDES_ID);
+	let crate_ = generate_crate_access();
 
 	Ok(quote!(
 		pub struct RuntimeApi {}
@@ -414,7 +429,7 @@ impl<'a> ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 	fn process(mut self, input: ItemImpl) -> ItemImpl {
 		let mut input = self.fold_item_impl(input);
 
-		let crate_ = generate_crate_access(HIDDEN_INCLUDES_ID);
+		let crate_ = generate_crate_access();
 
 		// Delete all functions, because all of them are default implemented by
 		// `decl_runtime_apis!`. We only need to implement the `__runtime_api_internal_call_api_at`
@@ -423,7 +438,7 @@ impl<'a> ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 		input.items.push(parse_quote! {
 			fn __runtime_api_internal_call_api_at(
 				&self,
-				at: <__SR_API_BLOCK__ as #crate_::BlockT>::Hash,
+				at: <__SrApiBlock__ as #crate_::BlockT>::Hash,
 				context: #crate_::ExecutionContext,
 				params: std::vec::Vec<u8>,
 				fn_name: &dyn Fn(#crate_::RuntimeVersion) -> &'static str,
@@ -435,7 +450,7 @@ impl<'a> ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 				}
 
 				let res = (|| {
-					let version = #crate_::CallApiAt::<__SR_API_BLOCK__>::runtime_version_at(
+					let version = #crate_::CallApiAt::<__SrApiBlock__>::runtime_version_at(
 						self.call,
 						at,
 					)?;
@@ -450,7 +465,7 @@ impl<'a> ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 						recorder: &self.recorder,
 					};
 
-					#crate_::CallApiAt::<__SR_API_BLOCK__>::call_api_at(
+					#crate_::CallApiAt::<__SrApiBlock__>::call_api_at(
 						self.call,
 						params,
 					)
@@ -469,7 +484,7 @@ impl<'a> ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 	fn fold_type_path(&mut self, input: TypePath) -> TypePath {
 		let new_ty_path =
-			if input == *self.runtime_block { parse_quote!(__SR_API_BLOCK__) } else { input };
+			if input == *self.runtime_block { parse_quote!(__SrApiBlock__) } else { input };
 
 		fold::fold_type_path(self, new_ty_path)
 	}
@@ -480,25 +495,26 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 		// Before we directly had the final block type and rust could determine that it is unwind
 		// safe, but now we just have a generic parameter `Block`.
 
-		let crate_ = generate_crate_access(HIDDEN_INCLUDES_ID);
+		let crate_ = generate_crate_access();
 
 		// Implement the trait for the `RuntimeApiImpl`
 		input.self_ty =
-			Box::new(parse_quote!( RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall> ));
+			Box::new(parse_quote!( RuntimeApiImpl<__SrApiBlock__, RuntimeApiImplCall> ));
 
 		input.generics.params.push(parse_quote!(
-			__SR_API_BLOCK__: #crate_::BlockT + std::panic::UnwindSafe +
+			__SrApiBlock__: #crate_::BlockT + std::panic::UnwindSafe +
 				std::panic::RefUnwindSafe
 		));
-		input.generics.params.push(
-			parse_quote!( RuntimeApiImplCall: #crate_::CallApiAt<__SR_API_BLOCK__> + 'static ),
-		);
+		input
+			.generics
+			.params
+			.push(parse_quote!( RuntimeApiImplCall: #crate_::CallApiAt<__SrApiBlock__> + 'static ));
 
 		let where_clause = input.generics.make_where_clause();
 
 		where_clause.predicates.push(parse_quote! {
 			RuntimeApiImplCall::StateBackend:
-				#crate_::StateBackend<#crate_::HashFor<__SR_API_BLOCK__>>
+				#crate_::StateBackend<#crate_::HashFor<__SrApiBlock__>>
 		});
 
 		where_clause.predicates.push(parse_quote! { &'static RuntimeApiImplCall: Send });
@@ -511,7 +527,7 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 		});
 
 		where_clause.predicates.push(parse_quote! {
-			__SR_API_BLOCK__::Header: std::panic::UnwindSafe + std::panic::RefUnwindSafe
+			__SrApiBlock__::Header: std::panic::UnwindSafe + std::panic::RefUnwindSafe
 		});
 
 		input.attrs = filter_cfg_attrs(&input.attrs);
@@ -574,7 +590,7 @@ fn generate_runtime_api_versions(impls: &[ItemImpl]) -> Result<TokenStream> {
 	let mut sections = Vec::<TokenStream>::with_capacity(impls.len());
 	let mut processed_traits = HashSet::new();
 
-	let c = generate_crate_access(HIDDEN_INCLUDES_ID);
+	let c = generate_crate_access();
 
 	for impl_ in impls {
 		let api_ver = extract_api_version(&impl_.attrs, impl_.span())?.map(|a| a as u32);
@@ -629,14 +645,11 @@ fn impl_runtime_apis_impl_inner(api_impls: &[ItemImpl]) -> Result<TokenStream> {
 	let dispatch_impl = generate_dispatch_function(api_impls)?;
 	let api_impls_for_runtime = generate_api_impl_for_runtime(api_impls)?;
 	let base_runtime_api = generate_runtime_api_base_structures()?;
-	let hidden_includes = generate_hidden_includes(HIDDEN_INCLUDES_ID);
 	let runtime_api_versions = generate_runtime_api_versions(api_impls)?;
 	let wasm_interface = generate_wasm_interface(api_impls)?;
 	let api_impls_for_runtime_api = generate_api_impl_for_runtime_api(api_impls)?;
 
-	Ok(quote!(
-		#hidden_includes
-
+	let impl_ = quote!(
 		#base_runtime_api
 
 		#api_impls_for_runtime
@@ -652,7 +665,15 @@ fn impl_runtime_apis_impl_inner(api_impls: &[ItemImpl]) -> Result<TokenStream> {
 
 			#wasm_interface
 		}
-	))
+	);
+
+	let impl_ = expander::Expander::new("impl_runtime_apis")
+		.dry(std::env::var("SP_API_EXPAND").is_err())
+		.verbose(true)
+		.write_to_out_dir(impl_)
+		.expect("Does not fail because of IO in OUT_DIR; qed");
+
+	Ok(impl_)
 }
 
 // Filters all attributes except the cfg ones.

substrate/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs

@@ -17,8 +17,8 @@
 
 use crate::utils::{
 	extract_block_type_from_trait_path, extract_impl_trait,
-	extract_parameter_names_types_and_borrows, generate_crate_access, generate_hidden_includes,
-	return_type_extract_type, AllowSelfRefInParameters, RequireQualifiedTraitPath,
+	extract_parameter_names_types_and_borrows, generate_crate_access, return_type_extract_type,
+	AllowSelfRefInParameters, RequireQualifiedTraitPath,
 };
 
 use proc_macro2::{Span, TokenStream};
@@ -33,9 +33,6 @@ use syn::{
 	Attribute, ItemImpl, Pat, Type, TypePath,
 };
 
-/// Unique identifier used to make the hidden includes unique for this macro.
-const HIDDEN_INCLUDES_ID: &str = "MOCK_IMPL_RUNTIME_APIS";
-
 /// The `advanced` attribute.
 ///
 /// If this attribute is given to a function, the function gets access to the `Hash` as first
@@ -65,7 +62,7 @@ impl Parse for RuntimeApiImpls {
 
 /// Implement the `ApiExt` trait and the `Core` runtime api.
 fn implement_common_api_traits(block_type: TypePath, self_ty: Type) -> Result<TokenStream> {
-	let crate_ = generate_crate_access(HIDDEN_INCLUDES_ID);
+	let crate_ = generate_crate_access();
 
 	Ok(quote!(
 		impl #crate_::ApiExt<#block_type> for #self_ty {
@@ -256,7 +253,7 @@ impl<'a> FoldRuntimeApiImpl<'a> {
 	fn process(mut self, impl_item: syn::ItemImpl) -> syn::ItemImpl {
 		let mut impl_item = self.fold_item_impl(impl_item);
 
-		let crate_ = generate_crate_access(HIDDEN_INCLUDES_ID);
+		let crate_ = generate_crate_access();
 
 		// We also need to overwrite all the `_with_context` methods. To do this,
 		// we clone all methods and add them again with the new name plus one more argument.
@@ -295,7 +292,7 @@ impl<'a> FoldRuntimeApiImpl<'a> {
 impl<'a> Fold for FoldRuntimeApiImpl<'a> {
 	fn fold_impl_item_method(&mut self, mut input: syn::ImplItemMethod) -> syn::ImplItemMethod {
 		let block = {
-			let crate_ = generate_crate_access(HIDDEN_INCLUDES_ID);
+			let crate_ = generate_crate_access();
 			let is_advanced = has_advanced_attribute(&mut input.attrs);
 			let mut errors = Vec::new();
 
@@ -469,14 +466,11 @@ pub fn mock_impl_runtime_apis_impl(input: proc_macro::TokenStream) -> proc_macro
 }
 
 fn mock_impl_runtime_apis_impl_inner(api_impls: &[ItemImpl]) -> Result<TokenStream> {
-	let hidden_includes = generate_hidden_includes(HIDDEN_INCLUDES_ID);
 	let GeneratedRuntimeApiImpls { impls, block_type, self_ty } =
 		generate_runtime_api_impls(api_impls)?;
 	let api_traits = implement_common_api_traits(block_type, self_ty)?;
 
 	Ok(quote!(
-		#hidden_includes
-
 		#impls
 
 		#api_traits

substrate/primitives/api/proc-macro/src/utils.rs

@@ -24,29 +24,19 @@ use syn::{
 
 use quote::{format_ident, quote};
 
-use std::env;
-
 use proc_macro_crate::{crate_name, FoundCrate};
 
 use crate::common::API_VERSION_ATTRIBUTE;
 
-fn generate_hidden_includes_mod_name(unique_id: &'static str) -> Ident {
-	Ident::new(&format!("sp_api_hidden_includes_{}", unique_id), Span::call_site())
-}
+use inflector::Inflector;
 
-/// Generates the hidden includes that are required to make the macro independent from its scope.
-pub fn generate_hidden_includes(unique_id: &'static str) -> TokenStream {
-	let mod_name = generate_hidden_includes_mod_name(unique_id);
+/// Generates the access to the `sc_client` crate.
+pub fn generate_crate_access() -> TokenStream {
 	match crate_name("sp-api") {
-		Ok(FoundCrate::Itself) => quote!(),
-		Ok(FoundCrate::Name(client_name)) => {
-			let client_name = Ident::new(&client_name, Span::call_site());
-			quote!(
-				#[doc(hidden)]
-				mod #mod_name {
-					pub extern crate #client_name as sp_api;
-				}
-			)
+		Ok(FoundCrate::Itself) => quote!(sp_api),
+		Ok(FoundCrate::Name(renamed_name)) => {
+			let renamed_name = Ident::new(&renamed_name, Span::call_site());
+			quote!(#renamed_name)
 		},
 		Err(e) => {
 			let err = Error::new(Span::call_site(), e).to_compile_error();
@@ -55,19 +45,12 @@ pub fn generate_hidden_includes(unique_id: &'static str) -> TokenStream {
 	}
 }
 
-/// Generates the access to the `sc_client` crate.
-pub fn generate_crate_access(unique_id: &'static str) -> TokenStream {
-	if env::var("CARGO_PKG_NAME").unwrap() == "sp-api" {
-		quote!(sp_api)
-	} else {
-		let mod_name = generate_hidden_includes_mod_name(unique_id);
-		quote!( self::#mod_name::sp_api )
-	}
-}
-
 /// Generates the name of the module that contains the trait declaration for the runtime.
 pub fn generate_runtime_mod_name_for_trait(trait_: &Ident) -> Ident {
-	Ident::new(&format!("runtime_decl_for_{}", trait_), Span::call_site())
+	Ident::new(
+		&format!("runtime_decl_for_{}", trait_.to_string().to_snake_case()),
+		Span::call_site(),
+	)
 }
 
 /// Get the type of a `syn::ReturnType`.