Allow to specify some max number of values for storages in pallet macro. (#8735)

* implement max_values + storages info

* some formatting + doc

* rename StoragesInfo -> PalletStorageInfo

* merge both StorageInfoTrait and PalletStorageInfo

I think it is more future proof. In the future some storage could make
use of multiple prefix. Like one to store how much value has been
inserted, etc...

* Update frame/support/procedural/src/storage/parse.rs

Co-authored-by: Peter Goodspeed-Niklaus <coriolinus@users.noreply.github.com>

* Update frame/support/procedural/src/storage/storage_struct.rs

Co-authored-by: Peter Goodspeed-Niklaus <coriolinus@users.noreply.github.com>

* Fix max_size using hasher information

hasher now expose `max_len` which allows to computes their maximum len.
For hasher without concatenation, it is the size of the hash part,
for hasher with concatenation, it is the size of the hash part + max
encoded len of the key.

* fix tests

* fix ui tests

Co-authored-by: Peter Goodspeed-Niklaus <coriolinus@users.noreply.github.com>

substrate/frame/support/procedural/src/storage/mod.rs

@@ -18,6 +18,7 @@
 //! `decl_storage` input definition and expansion.
 
 mod storage_struct;
+mod storage_info;
 mod parse;
 mod store_trait;
 mod getters;
@@ -35,6 +36,8 @@ use frame_support_procedural_tools::{
 
 /// All information contained in input of decl_storage
 pub struct DeclStorageDef {
+	/// Whether to generate the storage info
+	generate_storage_info: bool,
 	/// Name of the module used to import hidden imports.
 	hidden_crate: Option<syn::Ident>,
 	/// Visibility of store trait.
@@ -69,6 +72,8 @@ impl syn::parse::Parse for DeclStorageDef {
 
 /// Extended version of `DeclStorageDef` with useful precomputed value.
 pub struct DeclStorageDefExt {
+	/// Whether to generate the storage info
+	generate_storage_info: bool,
 	/// Name of the module used to import hidden imports.
 	hidden_crate: proc_macro2::TokenStream,
 	/// Hidden imports used by the module.
@@ -154,6 +159,7 @@ impl From<DeclStorageDef> for DeclStorageDefExt {
 		Self {
 			hidden_crate,
 			hidden_imports,
+			generate_storage_info: def.generate_storage_info,
 			visibility: def.visibility,
 			store_trait: def.store_trait,
 			module_name: def.module_name,
@@ -193,6 +199,8 @@ pub struct StorageLineDef {
 	getter: Option<syn::Ident>,
 	/// The name of the field to be used in genesis config if any.
 	config: Option<syn::Ident>,
+	/// The given max values with `max_values` attribute, or a none if not specified.
+	max_values: Option<syn::Expr>,
 	/// The build function of the storage if any.
 	build: Option<syn::Expr>,
 	/// Default value of genesis config field and also for storage when no value available.
@@ -210,6 +218,8 @@ pub struct StorageLineDefExt {
 	getter: Option<syn::Ident>,
 	/// The name of the field to be used in genesis config if any.
 	config: Option<syn::Ident>,
+	/// The given max values with `max_values` attribute, or a none if not specified.
+	max_values: Option<syn::Expr>,
 	/// The build function of the storage if any.
 	build: Option<syn::Expr>,
 	/// Default value of genesis config field and also for storage when no value available.
@@ -333,6 +343,7 @@ impl StorageLineDefExt {
 			name: storage_def.name,
 			getter: storage_def.getter,
 			config: storage_def.config,
+			max_values: storage_def.max_values,
 			build: storage_def.build,
 			default_value: storage_def.default_value,
 			storage_type: storage_def.storage_type,
@@ -469,6 +480,7 @@ pub fn decl_storage_impl(input: proc_macro::TokenStream) -> proc_macro::TokenStr
 	let instance_trait = instance_trait::decl_and_impl(&def_ext);
 	let genesis_config = genesis_config::genesis_config_and_build_storage(&def_ext);
 	let storage_struct = storage_struct::decl_and_impl(&def_ext);
+	let storage_info = storage_info::impl_storage_info(&def_ext);
 
 	quote!(
 		use #scrate::{
@@ -489,5 +501,6 @@ pub fn decl_storage_impl(input: proc_macro::TokenStream) -> proc_macro::TokenStr
 		#instance_trait
 		#genesis_config
 		#storage_struct
+		#storage_info
 	).into()
 }

substrate/frame/support/procedural/src/storage/parse.rs

@@ -21,10 +21,12 @@ use frame_support_procedural_tools::{ToTokens, Parse, syn_ext as ext};
 use syn::{Ident, Token, spanned::Spanned};
 
 mod keyword {
+	syn::custom_keyword!(generate_storage_info);
 	syn::custom_keyword!(hiddencrate);
 	syn::custom_keyword!(add_extra_genesis);
 	syn::custom_keyword!(extra_genesis_skip_phantom_data_field);
 	syn::custom_keyword!(config);
+	syn::custom_keyword!(max_values);
 	syn::custom_keyword!(build);
 	syn::custom_keyword!(get);
 	syn::custom_keyword!(map);
@@ -73,6 +75,7 @@ macro_rules! impl_parse_for_opt {
 /// Parsing usage only
 #[derive(Parse, ToTokens, Debug)]
 struct StorageDefinition {
+	pub generate_storage_info: Opt<GenerateStorageInfo>,
 	pub hidden_crate: Opt<SpecificHiddenCrate>,
 	pub visibility: syn::Visibility,
 	pub trait_token: Token![trait],
@@ -97,6 +100,12 @@ struct StorageDefinition {
 	pub extra_genesis: Opt<AddExtraGenesis>,
 }
 
+#[derive(Parse, ToTokens, Debug)]
+struct GenerateStorageInfo {
+	pub keyword: keyword::generate_storage_info,
+}
+impl_parse_for_opt!(GenerateStorageInfo => keyword::generate_storage_info);
+
 #[derive(Parse, ToTokens, Debug)]
 struct SpecificHiddenCrate {
 	pub keyword: keyword::hiddencrate,
@@ -160,6 +169,7 @@ struct DeclStorageLine {
 	pub name: Ident,
 	pub getter: Opt<DeclStorageGetter>,
 	pub config: Opt<DeclStorageConfig>,
+	pub max_values: Opt<DeclStorageMaxValues>,
 	pub build: Opt<DeclStorageBuild>,
 	pub coldot_token: Token![:],
 	pub storage_type: DeclStorageType,
@@ -188,6 +198,13 @@ struct DeclStorageConfig {
 
 impl_parse_for_opt!(DeclStorageConfig => keyword::config);
 
+#[derive(Parse, ToTokens, Debug)]
+struct DeclStorageMaxValues {
+	pub max_values_keyword: keyword::max_values,
+	pub expr: ext::Parens<syn::Expr>,
+}
+impl_parse_for_opt!(DeclStorageMaxValues => keyword::max_values);
+
 #[derive(Parse, ToTokens, Debug)]
 struct DeclStorageBuild {
 	pub build_keyword: keyword::build,
@@ -437,6 +454,7 @@ pub fn parse(input: syn::parse::ParseStream) -> syn::Result<super::DeclStorageDe
 	let storage_lines = parse_storage_line_defs(def.content.content.inner.into_iter())?;
 
 	Ok(super::DeclStorageDef {
+		generate_storage_info: def.generate_storage_info.inner.is_some(),
 		hidden_crate: def.hidden_crate.inner.map(|i| i.ident.content),
 		visibility: def.visibility,
 		module_name: def.module_ident,
@@ -490,6 +508,21 @@ fn parse_storage_line_defs(
 			})?;
 		}
 
+		let max_values = match &line.storage_type {
+			DeclStorageType::Map(_) | DeclStorageType::DoubleMap(_) | DeclStorageType::NMap(_) => {
+				line.max_values.inner.map(|i| i.expr.content)
+			},
+			DeclStorageType::Simple(_) => {
+				if let Some(max_values) = line.max_values.inner {
+					let msg = "unexpected max_values attribute for storage value.";
+					let span = max_values.max_values_keyword.span();
+					return Err(syn::Error::new(span, msg));
+				} else {
+					Some(syn::parse_quote!(1u32))
+				}
+			},
+		};
+
 		let span = line.storage_type.span();
 		let no_hasher_error = || syn::Error::new(
 			span,
@@ -534,6 +567,7 @@ fn parse_storage_line_defs(
 			name: line.name,
 			getter,
 			config,
+			max_values,
 			build: line.build.inner.map(|o| o.expr.content),
 			default_value: line.default_value.inner.map(|o| o.expr),
 			storage_type,

substrate/frame/support/procedural/src/storage/storage_info.rs

@@ -0,0 +1,57 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Implementation of trait `StorageInfoTrait` on module structure.
+
+use proc_macro2::TokenStream;
+use quote::quote;
+use super::DeclStorageDefExt;
+
+pub fn impl_storage_info(def: &DeclStorageDefExt) -> TokenStream {
+	if !def.generate_storage_info {
+		return Default::default()
+	}
+
+	let scrate = &def.hidden_crate;
+
+	let mut res_append_storage = TokenStream::new();
+
+	for line in def.storage_lines.iter() {
+		let storage_struct = &line.storage_struct;
+
+		res_append_storage.extend(quote!(
+			let mut storage_info = <
+				#storage_struct as #scrate::traits::StorageInfoTrait
+			>::storage_info();
+			res.append(&mut storage_info);
+		));
+	}
+
+	let module_struct = &def.module_struct;
+	let module_impl = &def.module_impl;
+	let where_clause = &def.where_clause;
+
+	quote!(
+		impl#module_impl #scrate::traits::StorageInfoTrait for #module_struct #where_clause {
+			fn storage_info() -> #scrate::sp_std::vec::Vec<#scrate::traits::StorageInfo> {
+				let mut res = #scrate::sp_std::vec![];
+				#res_append_storage
+				res
+			}
+		}
+	)
+}

substrate/frame/support/procedural/src/storage/storage_struct.rs

@@ -245,9 +245,167 @@ pub fn decl_and_impl(def: &DeclStorageDefExt) -> TokenStream {
 			}
 		};
 
+		let max_values = if let Some(max_values) = &line.max_values {
+			quote::quote!({
+				let max_values: u32 = (|| #max_values)();
+				Some(max_values)
+			})
+		} else {
+			quote::quote!(None)
+		};
+
+		let storage_info_impl = if def.generate_storage_info {
+			match &line.storage_type {
+				StorageLineTypeDef::Simple(_) => {
+					quote!(
+						impl<#impl_trait> #scrate::traits::StorageInfoTrait for #storage_struct
+						#optional_storage_where_clause
+						{
+							fn storage_info()
+								-> #scrate::sp_std::vec::Vec<#scrate::traits::StorageInfo>
+							{
+								use #scrate::sp_runtime::SaturatedConversion;
+
+								let max_size = <
+									#value_type as #scrate::traits::MaxEncodedLen
+								>::max_encoded_len()
+									.saturated_into();
+
+								#scrate::sp_std::vec![
+									#scrate::traits::StorageInfo {
+										prefix: <
+											#storage_struct as #scrate::#storage_generator_trait
+										>::storage_value_final_key(),
+										max_values: Some(1),
+										max_size: Some(max_size),
+									}
+								]
+							}
+						}
+					)
+				},
+				StorageLineTypeDef::Map(map) => {
+					let key = &map.key;
+					quote!(
+						impl<#impl_trait> #scrate::traits::StorageInfoTrait for #storage_struct
+						#optional_storage_where_clause
+						{
+							fn storage_info()
+								-> #scrate::sp_std::vec::Vec<#scrate::traits::StorageInfo>
+							{
+								use #scrate::sp_runtime::SaturatedConversion;
+								use #scrate::StorageHasher;
+
+								let key_max_size = <
+									Self as #scrate::storage::generator::StorageMap<_, _>
+								>::Hasher::max_len::<#key>();
+
+								let max_size = <
+									#value_type as #scrate::traits::MaxEncodedLen
+								>::max_encoded_len()
+									.saturating_add(key_max_size)
+									.saturated_into();
+
+								#scrate::sp_std::vec![
+									#scrate::traits::StorageInfo {
+										prefix: <
+											#storage_struct
+											as #scrate::storage::StoragePrefixedMap<#value_type>
+										>::final_prefix(),
+										max_values: #max_values,
+										max_size: Some(max_size),
+									}
+								]
+							}
+						}
+					)
+				},
+				StorageLineTypeDef::DoubleMap(map) => {
+					let key1 = &map.key1;
+					let key2 = &map.key2;
+					quote!(
+						impl<#impl_trait> #scrate::traits::StorageInfoTrait for #storage_struct
+						#optional_storage_where_clause
+						{
+							fn storage_info()
+								-> #scrate::sp_std::vec::Vec<#scrate::traits::StorageInfo>
+							{
+								use #scrate::sp_runtime::SaturatedConversion;
+								use #scrate::StorageHasher;
+
+								let key1_max_size = <
+									Self as #scrate::storage::generator::StorageDoubleMap<_, _, _>
+								>::Hasher1::max_len::<#key1>();
+
+								let key2_max_size = <
+									Self as #scrate::storage::generator::StorageDoubleMap<_, _, _>
+								>::Hasher2::max_len::<#key2>();
+
+								let max_size = <
+									#value_type as #scrate::traits::MaxEncodedLen
+								>::max_encoded_len()
+									.saturating_add(key1_max_size)
+									.saturating_add(key2_max_size)
+									.saturated_into();
+
+								#scrate::sp_std::vec![
+									#scrate::traits::StorageInfo {
+										prefix: <
+											#storage_struct
+											as #scrate::storage::StoragePrefixedMap<#value_type>
+										>::final_prefix(),
+										max_values: #max_values,
+										max_size: Some(max_size),
+									}
+								]
+							}
+						}
+					)
+				},
+				StorageLineTypeDef::NMap(map) => {
+					let key = &map.to_keygen_struct(scrate);
+					quote!(
+						impl<#impl_trait> #scrate::traits::StorageInfoTrait for #storage_struct
+						#optional_storage_where_clause
+						{
+							fn storage_info()
+								-> #scrate::sp_std::vec::Vec<#scrate::traits::StorageInfo>
+							{
+								use #scrate::sp_runtime::SaturatedConversion;
+
+								let key_max_size = <
+									#key as #scrate::storage::types::KeyGeneratorMaxEncodedLen
+								>::key_max_encoded_len();
+
+								let max_size = <
+									#value_type as #scrate::traits::MaxEncodedLen
+								>::max_encoded_len()
+									.saturating_add(key_max_size)
+									.saturated_into();
+
+								#scrate::sp_std::vec![
+									#scrate::traits::StorageInfo {
+										prefix: <
+											#storage_struct
+											as #scrate::storage::StoragePrefixedMap<#value_type>
+										>::final_prefix(),
+										max_values: #max_values,
+										max_size: Some(max_size),
+									}
+								]
+							}
+						}
+					)
+				},
+			}
+		} else {
+			TokenStream::default()
+		};
+
 		impls.extend(quote!(
 			#struct_decl
 			#struct_impl
+			#storage_info_impl
 		))
 	}