Construct Runtime v2 (#1378)

Moved from https://github.com/paritytech/substrate/pull/14788

----

Fixes https://github.com/paritytech/polkadot-sdk/issues/232

This PR introduces outer-macro approach for `construct_runtime` as
discussed in the linked issue. It looks like the following:
```rust
#[frame_support::runtime]
mod runtime {
	#[runtime::runtime]
        #[runtime::derive(
		RuntimeCall,
		RuntimeEvent,
		RuntimeError,
		RuntimeOrigin,
		RuntimeFreezeReason,
		RuntimeHoldReason,
		RuntimeSlashReason,
		RuntimeLockId,
                RuntimeTask,
	)]
	pub struct Runtime;

	#[runtime::pallet_index(0)]
	pub type System = frame_system;

	#[runtime::pallet_index(1)]
	pub type Timestamp = pallet_timestamp;

	#[runtime::pallet_index(2)]
	pub type Aura = pallet_aura;

	#[runtime::pallet_index(3)]
	pub type Grandpa = pallet_grandpa;

	#[runtime::pallet_index(4)]
	pub type Balances = pallet_balances;

	#[runtime::pallet_index(5)]
	pub type TransactionPayment = pallet_transaction_payment;

	#[runtime::pallet_index(6)]
	pub type Sudo = pallet_sudo;

	// Include the custom logic from the pallet-template in the runtime.
	#[runtime::pallet_index(7)]
	pub type TemplateModule = pallet_template;
}
```

## Features
- `#[runtime::runtime]` attached to a struct defines the main runtime
- `#[runtime::derive]` attached to this struct defines the types
generated by runtime
- `#[runtime::pallet_index]` must be attached to a pallet to define its
index
- `#[runtime::disable_call]` can be optionally attached to a pallet to
disable its calls
- `#[runtime::disable_unsigned]` can be optionally attached to a pallet
to disable unsigned calls
- A pallet instance can be defined as `TemplateModule:
pallet_template<Instance>`
- An optional attribute can be defined as
`#[frame_support::runtime(legacy_ordering)]` to ensure that the order of
hooks is same as the order of pallets (and not based on the
pallet_index). This is to support legacy runtimes and should be avoided
for new ones.

## Todo
- [x] Update the latest syntax in kitchensink and tests
- [x] Update UI tests
- [x] Docs

## Extension
- Abstract away the Executive similar to
https://github.com/paritytech/substrate/pull/14742
- Optionally avoid the need to specify all runtime types (TBD)

---------

Co-authored-by: Francisco Aguirre <franciscoaguirreperez@gmail.com>
Co-authored-by: Nikhil Gupta <>

substrate/frame/support/procedural/src/runtime/parse/helper.rs

@@ -0,0 +1,37 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::pallet::parse::helper::MutItemAttrs;
+use quote::ToTokens;
+
+pub(crate) fn take_first_item_runtime_attr<Attr>(
+	item: &mut impl MutItemAttrs,
+) -> syn::Result<Option<Attr>>
+where
+	Attr: syn::parse::Parse,
+{
+	let attrs = if let Some(attrs) = item.mut_item_attrs() { attrs } else { return Ok(None) };
+
+	if let Some(index) = attrs.iter().position(|attr| {
+		attr.path().segments.first().map_or(false, |segment| segment.ident == "runtime")
+	}) {
+		let runtime_attr = attrs.remove(index);
+		Ok(Some(syn::parse2(runtime_attr.into_token_stream())?))
+	} else {
+		Ok(None)
+	}
+}

substrate/frame/support/procedural/src/runtime/parse/mod.rs

@@ -0,0 +1,266 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod helper;
+pub mod pallet;
+pub mod pallet_decl;
+pub mod runtime_struct;
+pub mod runtime_types;
+
+use crate::construct_runtime::parse::Pallet;
+use pallet_decl::PalletDeclaration;
+use proc_macro2::TokenStream as TokenStream2;
+use quote::ToTokens;
+use std::collections::HashMap;
+use syn::{spanned::Spanned, Ident, Token};
+
+use frame_support_procedural_tools::syn_ext as ext;
+use runtime_types::RuntimeType;
+
+mod keyword {
+	use syn::custom_keyword;
+
+	custom_keyword!(runtime);
+	custom_keyword!(derive);
+	custom_keyword!(pallet_index);
+	custom_keyword!(disable_call);
+	custom_keyword!(disable_unsigned);
+}
+
+enum RuntimeAttr {
+	Runtime(proc_macro2::Span),
+	Derive(proc_macro2::Span, Vec<RuntimeType>),
+	PalletIndex(proc_macro2::Span, u8),
+	DisableCall(proc_macro2::Span),
+	DisableUnsigned(proc_macro2::Span),
+}
+
+impl RuntimeAttr {
+	fn span(&self) -> proc_macro2::Span {
+		match self {
+			Self::Runtime(span) => *span,
+			Self::Derive(span, _) => *span,
+			Self::PalletIndex(span, _) => *span,
+			Self::DisableCall(span) => *span,
+			Self::DisableUnsigned(span) => *span,
+		}
+	}
+}
+
+impl syn::parse::Parse for RuntimeAttr {
+	fn parse(input: syn::parse::ParseStream) -> syn::Result<Self> {
+		input.parse::<syn::Token![#]>()?;
+		let content;
+		syn::bracketed!(content in input);
+		content.parse::<keyword::runtime>()?;
+		content.parse::<syn::Token![::]>()?;
+
+		let lookahead = content.lookahead1();
+		if lookahead.peek(keyword::runtime) {
+			Ok(RuntimeAttr::Runtime(content.parse::<keyword::runtime>()?.span()))
+		} else if lookahead.peek(keyword::derive) {
+			let _ = content.parse::<keyword::derive>();
+			let derive_content;
+			syn::parenthesized!(derive_content in content);
+			let runtime_types =
+				derive_content.parse::<ext::Punctuated<RuntimeType, Token![,]>>()?;
+			let runtime_types = runtime_types.inner.into_iter().collect();
+			Ok(RuntimeAttr::Derive(derive_content.span(), runtime_types))
+		} else if lookahead.peek(keyword::pallet_index) {
+			let _ = content.parse::<keyword::pallet_index>();
+			let pallet_index_content;
+			syn::parenthesized!(pallet_index_content in content);
+			let pallet_index = pallet_index_content.parse::<syn::LitInt>()?;
+			if !pallet_index.suffix().is_empty() {
+				let msg = "Number literal must not have a suffix";
+				return Err(syn::Error::new(pallet_index.span(), msg))
+			}
+			Ok(RuntimeAttr::PalletIndex(pallet_index.span(), pallet_index.base10_parse()?))
+		} else if lookahead.peek(keyword::disable_call) {
+			Ok(RuntimeAttr::DisableCall(content.parse::<keyword::disable_call>()?.span()))
+		} else if lookahead.peek(keyword::disable_unsigned) {
+			Ok(RuntimeAttr::DisableUnsigned(content.parse::<keyword::disable_unsigned>()?.span()))
+		} else {
+			Err(lookahead.error())
+		}
+	}
+}
+
+#[derive(Debug, Clone)]
+pub enum AllPalletsDeclaration {
+	Implicit(ImplicitAllPalletsDeclaration),
+	Explicit(ExplicitAllPalletsDeclaration),
+}
+
+/// Declaration of a runtime with some pallet with implicit declaration of parts.
+#[derive(Debug, Clone)]
+pub struct ImplicitAllPalletsDeclaration {
+	pub name: Ident,
+	pub pallet_decls: Vec<PalletDeclaration>,
+	pub pallet_count: usize,
+}
+
+/// Declaration of a runtime with all pallet having explicit declaration of parts.
+#[derive(Debug, Clone)]
+pub struct ExplicitAllPalletsDeclaration {
+	pub name: Ident,
+	pub pallets: Vec<Pallet>,
+}
+
+pub struct Def {
+	pub input: TokenStream2,
+	pub item: syn::ItemMod,
+	pub runtime_struct: runtime_struct::RuntimeStructDef,
+	pub pallets: AllPalletsDeclaration,
+	pub runtime_types: Vec<RuntimeType>,
+}
+
+impl Def {
+	pub fn try_from(mut item: syn::ItemMod) -> syn::Result<Self> {
+		let input: TokenStream2 = item.to_token_stream().into();
+		let item_span = item.span();
+		let items = &mut item
+			.content
+			.as_mut()
+			.ok_or_else(|| {
+				let msg = "Invalid runtime definition, expected mod to be inlined.";
+				syn::Error::new(item_span, msg)
+			})?
+			.1;
+
+		let mut runtime_struct = None;
+		let mut runtime_types = None;
+
+		let mut indices = HashMap::new();
+		let mut names = HashMap::new();
+
+		let mut pallet_decls = vec![];
+		let mut pallets = vec![];
+
+		for item in items.iter_mut() {
+			let mut pallet_item = None;
+			let mut pallet_index = 0;
+
+			let mut disable_call = false;
+			let mut disable_unsigned = false;
+
+			while let Some(runtime_attr) =
+				helper::take_first_item_runtime_attr::<RuntimeAttr>(item)?
+			{
+				match runtime_attr {
+					RuntimeAttr::Runtime(span) if runtime_struct.is_none() => {
+						let p = runtime_struct::RuntimeStructDef::try_from(span, item)?;
+						runtime_struct = Some(p);
+					},
+					RuntimeAttr::Derive(_, types) if runtime_types.is_none() => {
+						runtime_types = Some(types);
+					},
+					RuntimeAttr::PalletIndex(span, index) => {
+						pallet_index = index;
+						pallet_item = if let syn::Item::Type(item) = item {
+							Some(item.clone())
+						} else {
+							let msg = "Invalid runtime::pallet_index, expected type definition";
+							return Err(syn::Error::new(span, msg))
+						};
+					},
+					RuntimeAttr::DisableCall(_) => disable_call = true,
+					RuntimeAttr::DisableUnsigned(_) => disable_unsigned = true,
+					attr => {
+						let msg = "Invalid duplicated attribute";
+						return Err(syn::Error::new(attr.span(), msg))
+					},
+				}
+			}
+
+			if let Some(pallet_item) = pallet_item {
+				match *pallet_item.ty.clone() {
+					syn::Type::Path(ref path) => {
+						let pallet_decl =
+							PalletDeclaration::try_from(item.span(), &pallet_item, path)?;
+
+						if let Some(used_pallet) =
+							names.insert(pallet_decl.name.clone(), pallet_decl.name.span())
+						{
+							let msg = "Two pallets with the same name!";
+
+							let mut err = syn::Error::new(used_pallet, &msg);
+							err.combine(syn::Error::new(pallet_decl.name.span(), &msg));
+							return Err(err)
+						}
+
+						pallet_decls.push(pallet_decl);
+					},
+					syn::Type::TraitObject(syn::TypeTraitObject { bounds, .. }) => {
+						let pallet = Pallet::try_from(
+							item.span(),
+							&pallet_item,
+							pallet_index,
+							disable_call,
+							disable_unsigned,
+							&bounds,
+						)?;
+
+						if let Some(used_pallet) = indices.insert(pallet.index, pallet.name.clone())
+						{
+							let msg = format!(
+								"Pallet indices are conflicting: Both pallets {} and {} are at index {}",
+								used_pallet, pallet.name, pallet.index,
+							);
+							let mut err = syn::Error::new(used_pallet.span(), &msg);
+							err.combine(syn::Error::new(pallet.name.span(), msg));
+							return Err(err)
+						}
+
+						pallets.push(pallet);
+					},
+					_ => continue,
+				}
+			}
+		}
+
+		let name = item.ident.clone();
+		let decl_count = pallet_decls.len();
+		let pallets = if decl_count > 0 {
+			AllPalletsDeclaration::Implicit(ImplicitAllPalletsDeclaration {
+				name,
+				pallet_decls,
+				pallet_count: decl_count.saturating_add(pallets.len()),
+			})
+		} else {
+			AllPalletsDeclaration::Explicit(ExplicitAllPalletsDeclaration { name, pallets })
+		};
+
+		let def = Def {
+			input,
+			item,
+			runtime_struct: runtime_struct.ok_or_else(|| {
+				syn::Error::new(item_span,
+					"Missing Runtime. Please add a struct inside the module and annotate it with `#[runtime::runtime]`"
+				)
+			})?,
+			pallets,
+			runtime_types: runtime_types.ok_or_else(|| {
+				syn::Error::new(item_span,
+					"Missing Runtime Types. Please annotate the runtime struct with `#[runtime::derive]`"
+				)
+			})?,
+		};
+
+		Ok(def)
+	}
+}

substrate/frame/support/procedural/src/runtime/parse/pallet.rs

@@ -0,0 +1,99 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::construct_runtime::parse::{Pallet, PalletPart, PalletPartKeyword, PalletPath};
+use quote::ToTokens;
+use syn::{punctuated::Punctuated, spanned::Spanned, token, Error, Ident, PathArguments};
+
+impl Pallet {
+	pub fn try_from(
+		attr_span: proc_macro2::Span,
+		item: &syn::ItemType,
+		pallet_index: u8,
+		disable_call: bool,
+		disable_unsigned: bool,
+		bounds: &Punctuated<syn::TypeParamBound, token::Plus>,
+	) -> syn::Result<Self> {
+		let name = item.ident.clone();
+
+		let mut pallet_path = None;
+		let mut pallet_parts = vec![];
+
+		for (index, bound) in bounds.into_iter().enumerate() {
+			if let syn::TypeParamBound::Trait(syn::TraitBound { path, .. }) = bound {
+				if index == 0 {
+					pallet_path = Some(PalletPath { inner: path.clone() });
+				} else {
+					let pallet_part = syn::parse2::<PalletPart>(bound.into_token_stream())?;
+					pallet_parts.push(pallet_part);
+				}
+			} else {
+				return Err(Error::new(
+					attr_span,
+					"Invalid pallet declaration, expected a path or a trait object",
+				))
+			};
+		}
+
+		let mut path = pallet_path.ok_or(Error::new(
+			attr_span,
+			"Invalid pallet declaration, expected a path or a trait object",
+		))?;
+
+		let mut instance = None;
+		if let Some(segment) = path.inner.segments.iter_mut().find(|seg| !seg.arguments.is_empty())
+		{
+			if let PathArguments::AngleBracketed(syn::AngleBracketedGenericArguments {
+				args, ..
+			}) = segment.arguments.clone()
+			{
+				if let Some(syn::GenericArgument::Type(syn::Type::Path(arg_path))) = args.first() {
+					instance =
+						Some(Ident::new(&arg_path.to_token_stream().to_string(), arg_path.span()));
+					segment.arguments = PathArguments::None;
+				}
+			}
+		}
+
+		pallet_parts = pallet_parts
+			.into_iter()
+			.filter(|part| {
+				if let (true, &PalletPartKeyword::Call(_)) = (disable_call, &part.keyword) {
+					false
+				} else if let (true, &PalletPartKeyword::ValidateUnsigned(_)) =
+					(disable_unsigned, &part.keyword)
+				{
+					false
+				} else {
+					true
+				}
+			})
+			.collect();
+
+		let cfg_pattern = vec![];
+
+		Ok(Pallet {
+			is_expanded: true,
+			name,
+			index: pallet_index,
+			path,
+			instance,
+			cfg_pattern,
+			pallet_parts,
+		})
+	}
+}

substrate/frame/support/procedural/src/runtime/parse/pallet_decl.rs

@@ -0,0 +1,60 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use quote::ToTokens;
+use syn::{spanned::Spanned, Attribute, Ident, PathArguments};
+
+/// The declaration of a pallet.
+#[derive(Debug, Clone)]
+pub struct PalletDeclaration {
+	/// The name of the pallet, e.g.`System` in `System: frame_system`.
+	pub name: Ident,
+	/// Optional attributes tagged right above a pallet declaration.
+	pub attrs: Vec<Attribute>,
+	/// The path of the pallet, e.g. `frame_system` in `System: frame_system`.
+	pub path: syn::Path,
+	/// The instance of the pallet, e.g. `Instance1` in `Council: pallet_collective::<Instance1>`.
+	pub instance: Option<Ident>,
+}
+
+impl PalletDeclaration {
+	pub fn try_from(
+		_attr_span: proc_macro2::Span,
+		item: &syn::ItemType,
+		path: &syn::TypePath,
+	) -> syn::Result<Self> {
+		let name = item.ident.clone();
+
+		let mut path = path.path.clone();
+
+		let mut instance = None;
+		if let Some(segment) = path.segments.iter_mut().find(|seg| !seg.arguments.is_empty()) {
+			if let PathArguments::AngleBracketed(syn::AngleBracketedGenericArguments {
+				args, ..
+			}) = segment.arguments.clone()
+			{
+				if let Some(syn::GenericArgument::Type(syn::Type::Path(arg_path))) = args.first() {
+					instance =
+						Some(Ident::new(&arg_path.to_token_stream().to_string(), arg_path.span()));
+					segment.arguments = PathArguments::None;
+				}
+			}
+		}
+
+		Ok(Self { name, path, instance, attrs: item.attrs.clone() })
+	}
+}

substrate/frame/support/procedural/src/runtime/parse/runtime_struct.rs

@@ -0,0 +1,35 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use syn::spanned::Spanned;
+pub struct RuntimeStructDef {
+	pub ident: syn::Ident,
+	pub attr_span: proc_macro2::Span,
+}
+
+impl RuntimeStructDef {
+	pub fn try_from(attr_span: proc_macro2::Span, item: &mut syn::Item) -> syn::Result<Self> {
+		let item = if let syn::Item::Struct(item) = item {
+			item
+		} else {
+			let msg = "Invalid runtime::runtime, expected struct definition";
+			return Err(syn::Error::new(item.span(), msg))
+		};
+
+		Ok(Self { ident: item.ident.clone(), attr_span })
+	}
+}

substrate/frame/support/procedural/src/runtime/parse/runtime_types.rs

@@ -0,0 +1,76 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use syn::{
+	parse::{Parse, ParseStream},
+	Result,
+};
+
+mod keyword {
+	use syn::custom_keyword;
+
+	custom_keyword!(RuntimeCall);
+	custom_keyword!(RuntimeEvent);
+	custom_keyword!(RuntimeError);
+	custom_keyword!(RuntimeOrigin);
+	custom_keyword!(RuntimeFreezeReason);
+	custom_keyword!(RuntimeHoldReason);
+	custom_keyword!(RuntimeSlashReason);
+	custom_keyword!(RuntimeLockId);
+	custom_keyword!(RuntimeTask);
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum RuntimeType {
+	RuntimeCall(keyword::RuntimeCall),
+	RuntimeEvent(keyword::RuntimeEvent),
+	RuntimeError(keyword::RuntimeError),
+	RuntimeOrigin(keyword::RuntimeOrigin),
+	RuntimeFreezeReason(keyword::RuntimeFreezeReason),
+	RuntimeHoldReason(keyword::RuntimeHoldReason),
+	RuntimeSlashReason(keyword::RuntimeSlashReason),
+	RuntimeLockId(keyword::RuntimeLockId),
+	RuntimeTask(keyword::RuntimeTask),
+}
+
+impl Parse for RuntimeType {
+	fn parse(input: ParseStream) -> Result<Self> {
+		let lookahead = input.lookahead1();
+
+		if lookahead.peek(keyword::RuntimeCall) {
+			Ok(Self::RuntimeCall(input.parse()?))
+		} else if lookahead.peek(keyword::RuntimeEvent) {
+			Ok(Self::RuntimeEvent(input.parse()?))
+		} else if lookahead.peek(keyword::RuntimeError) {
+			Ok(Self::RuntimeError(input.parse()?))
+		} else if lookahead.peek(keyword::RuntimeOrigin) {
+			Ok(Self::RuntimeOrigin(input.parse()?))
+		} else if lookahead.peek(keyword::RuntimeFreezeReason) {
+			Ok(Self::RuntimeFreezeReason(input.parse()?))
+		} else if lookahead.peek(keyword::RuntimeHoldReason) {
+			Ok(Self::RuntimeHoldReason(input.parse()?))
+		} else if lookahead.peek(keyword::RuntimeSlashReason) {
+			Ok(Self::RuntimeSlashReason(input.parse()?))
+		} else if lookahead.peek(keyword::RuntimeLockId) {
+			Ok(Self::RuntimeLockId(input.parse()?))
+		} else if lookahead.peek(keyword::RuntimeTask) {
+			Ok(Self::RuntimeTask(input.parse()?))
+		} else {
+			Err(lookahead.error())
+		}
+	}
+}