[FRAME Core] Adds ability to split a pallet across multiple files (#13950)

* Initial setup * Updates macro_magic version and refactors accordingly * Removes unwrap from macro * Splits into multiple sections * Uses call_site to fix macro hygiene issue * Initial setup * Removes unnecessary changes * Moves template palet back * Updates cargo.lock * Moves BagsList inside mod * Comments access to internal functions for now * Updates tests * Uncomments code * Fixes test * Moves bags-list to separate crate * Initial setup * Removes bags-list changes * Fix structure * Minor update * Addresses review comment * Adds a couple of UI tests. More to be added * Adds err files * Adds test for no pallet * Adds doc * Updates versions * Adds benchmarking * Updates doc link * ".git/.scripts/commands/fmt/fmt.sh" * Minor update * Adds missing changes * ".git/.scripts/commands/fmt/fmt.sh" * Update frame/support/procedural/src/lib.rs Co-authored-by: Sam Johnson <sam@durosoft.com> * Addresses review comments * Addresses review comments * ".git/.scripts/commands/fmt/fmt.sh" * Update frame/support/procedural/src/lib.rs Co-authored-by: Sam Johnson <sam@durosoft.com> * Update frame/support/procedural/src/lib.rs Co-authored-by: Sam Johnson <sam@durosoft.com> * Update frame/support/procedural/src/lib.rs Co-authored-by: Sam Johnson <sam@durosoft.com> * Adds UI test for disambiguation * ".git/.scripts/commands/fmt/fmt.sh" * Makes clippy happy * ".git/.scripts/commands/fmt/fmt.sh" * Fixes frame support test * Fixes frame support test * Split items other than storage * Updates versions * Fixes some review comments * Addresses review comments * ".git/.scripts/commands/fmt/fmt.sh" * Updates docs * Adds experimental disclaimer * ".git/.scripts/commands/fmt/fmt.sh" * Update frame/support/test/tests/split_ui/no_section_found.rs Co-authored-by: Sam Johnson <sam@durosoft.com> * Addresses review comments * Fixes test --------- Co-authored-by: command-bot <> Co-authored-by: command-bot <ci@gitlab.parity.io> Co-authored-by: Sam Johnson <sam@durosoft.com>
2026-06-24 02:01:06 +00:00 · 2023-06-27 17:42:03 +05:30
parent 13cb7ccc57
commit 5d8774016c
23 changed files with 837 additions and 5 deletions
@@ -43,7 +43,7 @@ use proc_macro::TokenStream;
 use quote::{quote, ToTokens};
 use std::{cell::RefCell, str::FromStr};
 pub(crate) use storage::INHERENT_INSTANCE_NAME;
-use syn::{parse_macro_input, ItemImpl};
+use syn::{parse_macro_input, Error, ItemImpl, ItemMod};

 thread_local! {
 	/// A global counter, can be used to generate a relatively unique identifier.
@@ -1761,3 +1761,113 @@ pub fn origin(_: TokenStream, _: TokenStream) -> TokenStream {
 pub fn composite_enum(_: TokenStream, _: TokenStream) -> TokenStream {
 	pallet_macro_stub()
 }
+
+/// Can be attached to a module. Doing so will declare that module as importable into a pallet
+/// via [`#[import_section]`](`macro@import_section`).
+///
+/// Note that sections are imported by their module name/ident, and should be referred to by
+/// their _full path_ from the perspective of the target pallet. Do not attempt to make use
+/// of `use` statements to bring pallet sections into scope, as this will not work (unless
+/// you do so as part of a wildcard import, in which case it will work).
+///
+/// ## Naming Logistics
+///
+/// Also note that because of how `#[pallet_section]` works, pallet section names must be
+/// globally unique _within the crate in which they are defined_. For more information on
+/// why this must be the case, see macro_magic's
+/// [`#[export_tokens]`](https://docs.rs/macro_magic/latest/macro_magic/attr.export_tokens.html) macro.
+///
+/// Optionally, you may provide an argument to `#[pallet_section]` such as
+/// `#[pallet_section(some_ident)]`, in the event that there is another pallet section in
+/// same crate with the same ident/name. The ident you specify can then be used instead of
+/// the module's ident name when you go to import it via `#[import_section]`.
+#[proc_macro_attribute]
+pub fn pallet_section(attr: TokenStream, tokens: TokenStream) -> TokenStream {
+	let tokens_clone = tokens.clone();
+	// ensure this can only be attached to a module
+	let _mod = parse_macro_input!(tokens_clone as ItemMod);
+
+	// use macro_magic's export_tokens as the internal implementation otherwise
+	match macro_magic::mm_core::export_tokens_internal(attr, tokens, false) {
+		Ok(tokens) => tokens.into(),
+		Err(err) => err.to_compile_error().into(),
+	}
+}
+
+/// An attribute macro that can be attached to a module declaration. Doing so will
+/// Imports the contents of the specified external pallet section that was defined
+/// previously using [`#[pallet_section]`](`macro@pallet_section`).
+///
+/// ## Example
+/// ```ignore
+/// #[import_section(some_section)]
+/// #[pallet]
+/// pub mod pallet {
+///     // ...
+/// }
+/// ```
+/// where `some_section` was defined elsewhere via:
+/// ```ignore
+/// #[pallet_section]
+/// pub mod some_section {
+///     // ...
+/// }
+/// ```
+///
+/// This will result in the contents of `some_section` being _verbatim_ imported into
+/// the pallet above. Note that since the tokens for `some_section` are essentially
+/// copy-pasted into the target pallet, you cannot refer to imports that don't also
+/// exist in the target pallet, but this is easily resolved by including all relevant
+/// `use` statements within your pallet section, so they are imported as well, or by
+/// otherwise ensuring that you have the same imports on the target pallet.
+///
+/// It is perfectly permissible to import multiple pallet sections into the same pallet,
+/// which can be done by having multiple `#[import_section(something)]` attributes
+/// attached to the pallet.
+///
+/// Note that sections are imported by their module name/ident, and should be referred to by
+/// their _full path_ from the perspective of the target pallet.
+#[import_tokens_attr {
+    format!(
+        "{}::macro_magic",
+        match generate_crate_access_2018("frame-support") {
+            Ok(path) => Ok(path),
+            Err(_) => generate_crate_access_2018("frame"),
+        }
+        .expect("Failed to find either `frame-support` or `frame` in `Cargo.toml` dependencies.")
+        .to_token_stream()
+        .to_string()
+    )
+}]
+#[proc_macro_attribute]
+pub fn import_section(attr: TokenStream, tokens: TokenStream) -> TokenStream {
+	let foreign_mod = parse_macro_input!(attr as ItemMod);
+	let mut internal_mod = parse_macro_input!(tokens as ItemMod);
+
+	// check that internal_mod is a pallet module
+	if !internal_mod.attrs.iter().any(|attr| {
+		if let Some(last_seg) = attr.path().segments.last() {
+			last_seg.ident == "pallet"
+		} else {
+			false
+		}
+	}) {
+		return Error::new(
+			internal_mod.ident.span(),
+			"`#[import_section]` can only be applied to a valid pallet module",
+		)
+		.to_compile_error()
+		.into()
+	}
+
+	if let Some(ref mut content) = internal_mod.content {
+		if let Some(foreign_content) = foreign_mod.content {
+			content.1.extend(foreign_content.1);
+		}
+	}
+
+	quote! {
+		#internal_mod
+	}
+	.into()
+}
@@ -2896,9 +2896,9 @@ pub mod pallet_macros {
 	pub use frame_support_procedural::{
 		call_index, compact, composite_enum, config, constant,
 		disable_frame_system_supertrait_check, error, event, extra_constants, generate_deposit,
-		generate_store, genesis_build, genesis_config, getter, hooks, inherent, no_default, origin,
-		storage, storage_prefix, storage_version, type_value, unbounded, validate_unsigned, weight,
-		whitelist_storage,
+		generate_store, genesis_build, genesis_config, getter, hooks, import_section, inherent,
+		no_default, origin, pallet_section, storage, storage_prefix, storage_version, type_value,
+		unbounded, validate_unsigned, weight, whitelist_storage,
 	};
 }

@@ -0,0 +1,36 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#[rustversion::attr(not(stable), ignore)]
+#[cfg(not(feature = "disable-ui-tests"))]
+#[test]
+fn split_ui() {
+	// Only run the ui tests when `RUN_UI_TESTS` is set.
+	if std::env::var("RUN_UI_TESTS").is_err() {
+		return
+	}
+
+	// As trybuild is using `cargo check`, we don't need the real WASM binaries.
+	std::env::set_var("SKIP_WASM_BUILD", "1");
+
+	// Deny all warnings since we emit warnings as part of a Pallet's UI.
+	std::env::set_var("RUSTFLAGS", "--deny warnings");
+
+	let t = trybuild::TestCases::new();
+	t.compile_fail("tests/split_ui/*.rs");
+	t.pass("tests/split_ui/pass/*.rs");
+}
@@ -0,0 +1,17 @@
+#![cfg_attr(not(feature = "std"), no_std)]
+
+use frame_support::pallet_macros::*;
+
+#[pallet_section]
+mod storages {
+	#[pallet::storage]
+	pub type MyStorageMap<T: Config> = StorageMap<_, _, u32, u64>;
+}
+
+#[import_section(storages)]
+pub mod pallet {
+	
+}
+
+fn main() {
+}
@@ -0,0 +1,5 @@
+error: `#[import_section]` can only be applied to a valid pallet module
+  --> tests/split_ui/import_without_pallet.rs:12:9
+   |
+12 | pub mod pallet {
+   |         ^^^^^^
@@ -0,0 +1,29 @@
+#![cfg_attr(not(feature = "std"), no_std)]
+
+use frame_support::pallet_macros::*;
+
+pub use pallet::*;
+
+#[import_section(storages_dev)]
+#[frame_support::pallet(dev_mode)]
+pub mod pallet {
+	use frame_support::pallet_prelude::*;
+	use frame_system::pallet_prelude::*;
+
+	#[pallet::pallet]
+	pub struct Pallet<T>(_);
+
+	#[pallet::config]
+	pub trait Config: frame_system::Config {}
+
+	#[pallet::call]
+	impl<T: Config> Pallet<T> {
+		pub fn my_call(_origin: OriginFor<T>) -> DispatchResult {
+			MyStorageMap::<T>::insert(1, 2);
+			Ok(())
+		}
+	}
+}
+
+fn main() {
+}
@@ -0,0 +1,13 @@
+error[E0432]: unresolved import `pallet`
+ --> tests/split_ui/no_section_found.rs:5:9
+  |
+5 | pub use pallet::*;
+  |         ^^^^^^ help: a similar path exists: `test_pallet::pallet`
+
+error: cannot find macro `__export_tokens_tt_storages_dev` in this scope
+ --> tests/split_ui/no_section_found.rs:7:1
+  |
+7 | #[import_section(storages_dev)]
+  | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  |
+  = note: this error originates in the macro `frame_support::macro_magic::forward_tokens` (in Nightly builds, run with -Z macro-backtrace for more info)
@@ -0,0 +1,40 @@
+#![cfg_attr(not(feature = "std"), no_std)]
+
+use frame_support::pallet_macros::*;
+
+pub use pallet::*;
+
+#[pallet_section]
+mod events {
+	#[pallet::event]
+	#[pallet::generate_deposit(pub(super) fn deposit_event)]
+	pub enum Event<T: Config> {
+		SomethingDone,
+	}
+}
+
+#[import_section(events)]
+#[frame_support::pallet(dev_mode)]
+pub mod pallet {
+	use frame_support::pallet_prelude::*;
+	use frame_system::pallet_prelude::*;
+
+	#[pallet::pallet]
+	pub struct Pallet<T>(_);
+
+	#[pallet::config]
+	pub trait Config: frame_system::Config {
+		type RuntimeEvent: From<Event<Self>> + IsType<<Self as frame_system::Config>::RuntimeEvent>;
+	}
+
+	#[pallet::call]
+	impl<T: Config> Pallet<T> {
+		pub fn my_call(_origin: OriginFor<T>) -> DispatchResult {
+			Self::deposit_event(Event::SomethingDone);
+			Ok(())
+		}
+	}
+}
+
+fn main() {
+}
@@ -0,0 +1,61 @@
+#![cfg_attr(not(feature = "std"), no_std)]
+
+use frame_support::pallet_macros::*;
+
+pub use pallet::*;
+
+mod first {
+	use super::*;
+
+	#[pallet_section]
+	mod section {
+		#[pallet::event]
+		#[pallet::generate_deposit(pub(super) fn deposit_event)]
+		pub enum Event<T: Config> {
+			SomethingDone,
+		}
+	}
+}
+
+mod second {
+	use super::*;
+	
+	#[pallet_section(section2)]
+	mod section {
+		#[pallet::error]
+		pub enum Error<T> {
+			NoneValue,
+		}
+	}
+}
+
+#[import_section(first::section)]
+#[import_section(second::section2)]
+#[frame_support::pallet(dev_mode)]
+pub mod pallet {
+	use frame_support::pallet_prelude::*;
+	use frame_system::pallet_prelude::*;
+
+	#[pallet::pallet]
+	pub struct Pallet<T>(_);
+
+	#[pallet::config]
+	pub trait Config: frame_system::Config {
+		type RuntimeEvent: From<Event<Self>> + IsType<<Self as frame_system::Config>::RuntimeEvent>;
+	}
+
+	#[pallet::call]
+	impl<T: Config> Pallet<T> {
+		pub fn my_call(_origin: OriginFor<T>) -> DispatchResult {
+			Self::deposit_event(Event::SomethingDone);
+			Ok(())
+		}
+
+		pub fn my_call_2(_origin: OriginFor<T>) -> DispatchResult {
+			return Err(Error::<T>::NoneValue.into())
+		}
+	}
+}
+
+fn main() {
+}
@@ -0,0 +1,34 @@
+#![cfg_attr(not(feature = "std"), no_std)]
+
+use frame_support::pallet_macros::*;
+
+pub use pallet::*;
+
+#[pallet_section]
+mod storages {
+	#[pallet::storage]
+	pub type MyStorageMap<T: Config> = StorageMap<_, _, u32, u64>;
+}
+
+#[frame_support::pallet(dev_mode)]
+pub mod pallet {
+	use frame_support::pallet_prelude::*;
+	use frame_system::pallet_prelude::*;
+
+	#[pallet::pallet]
+	pub struct Pallet<T>(_);
+
+	#[pallet::config]
+	pub trait Config: frame_system::Config {}
+
+	#[pallet::call]
+	impl<T: Config> Pallet<T> {
+		pub fn my_call(_origin: OriginFor<T>) -> DispatchResult {
+			MyStorageMap::<T>::insert(1, 2);
+			Ok(())
+		}
+	}
+}
+
+fn main() {
+}
@@ -0,0 +1,8 @@
+error[E0433]: failed to resolve: use of undeclared type `MyStorageMap`
+  --> tests/split_ui/section_not_imported.rs:27:4
+   |
+27 |             MyStorageMap::<T>::insert(1, 2);
+   |             ^^^^^^^^^^^^
+   |             |
+   |             use of undeclared type `MyStorageMap`
+   |             help: a struct with a similar name exists: `StorageMap`